amadan

carlos / platform Public

feat: activator, releases/channels, fleet ops (ported from Eleven + titogo)

Activator: hibernation state machine with exec backend, sidecar
co-spawn, due-time wakes — Eleven's 24 tests ported wholesale.
Releases: content-addressed manifests, signed CAS channel pointers,
bake windows (rollback never bakes), adopt pass with symlink-only
for hibernating hosts. Ops: fleet litestream config gen (skips
hibernating routes) + restore-verify.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell 845ffd40e7fb55c9ec4055958f428368e148a5d8 parent 9be1c80
16 files changed, +6114 −0
  • internal/activator/activator.go +1064 −0
  • internal/activator/activator_test.go +1841 −0
  • internal/activator/backend_exec.go +435 −0
  • internal/ops/litestream.go +210 −0
  • internal/ops/ops.go +139 −0
  • internal/ops/ops_test.go +144 −0
  • internal/release/adopt.go +217 −0
  • internal/release/adopt_test.go +193 −0
  • internal/release/bake.go +179 −0
  • internal/release/bake_test.go +171 −0
  • internal/release/release.go +477 −0
  • internal/release/release_test.go +279 −0
  • internal/release/store.go +280 −0
  • internal/release/store_test.go +137 −0
  • internal/release/versions.go +236 −0
  • internal/release/versions_test.go +112 −0
diff --git a/internal/activator/activator.go b/internal/activator/activator.go
new file mode 100644
index 000000000..17615c317
--- /dev/null
+++ b/internal/activator/activator.go
@@ -0,0 +1,1064 @@
+// Package activator is the hibernation state machine: instances hibernate.
+// When the agent is given an S3 bucket (CARLOS_HIBERNATE_BUCKET) it stops
+// relying on systemd to keep instance processes alive and instead runs them
+// as its own children, on demand — "units are for the edge, instances are its
+// children". A request for a host with no live process claims the per-host S3
+// single-writer lease, restores the DB from S3 if the local copy is missing,
+// spawns the instance, and proxies; after an idle window the instance is
+// checkpointed back to S3, the lease released, and the process stopped. State
+// parks in S3; processes exist only while someone's talking.
+//
+// Ported from lchat activator.go (Eleven's activator, issue #550 Part 4 v2),
+// with the mesh (multi-box placement: gossip/HRW, peer proxying) dropped for
+// single-box v1 — `// mesh seam:` comments mark every point where it slots
+// back in. Membership of one: hash picks self, leases are uncontested except
+// against a previous incarnation of ourselves.
+//
+// Sidecar co-spawn: a host with a sidecar staged (/data/<host>-sidecar.env)
+// gets its `sidecar run` child spawned right after the instance turns
+// healthy, and SIGTERMed first at sleep — the sidecar exists exactly while
+// its instance is awake, so its lifecycle solves itself. Its own connections
+// flow back through the edge (the box pins its hosted names to loopback), so
+// the sidecar marks them (SidecarHeader) and the activator excludes them from
+// the idle clock — otherwise the pair could never sleep, and an orphaned
+// sidecar's reconnect loop would wake the instance right back (see ServeHTTP).
+//
+// Opt-in is PER HOST (routes.hibernate): with the bucket env set, only
+// flagged instances are activator-managed; the rest stay on their systemd
+// units and are proxied straight. That's what makes a production canary safe —
+// the env ships fleet-wide as a no-op, then flipping one host's flag migrates
+// exactly one instance: its unit stops, the flag flips, and the first request
+// wakes it here.
+//
+// The due-time index: scheduled server-side work fires on time even while its
+// instance sleeps. At sleep the stop path asks the instance for its earliest
+// pending delivery (GET /api/next-due over the socket, admin-gated) and
+// records it in the registry (routes.next_due — persisted so it survives
+// agent restarts); the sweeper wakes any sleeping host whose time has
+// arrived, with no client request. Once up, the instance's own scheduler
+// delivers, and the normal idle window puts it back to sleep — which
+// re-records the NEXT due time, so the cycle maintains itself. Worst-case
+// lateness ≈ one activator sweep interval (≤30s) + spawn + one scheduler tick.
+package activator
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log"
+ "net/http"
+ "os"
+ "path"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// SidecarHeader marks every request a co-spawned sidecar makes to its own
+// instance through the edge. Marked traffic is activity-invisible: served
+// while the instance is awake, refused (503) while it isn't, never a wake.
+// The marker is deliberately spoofable-but-harmless: it only DOWNGRADES the
+// sender's own requests (no activity credit; 503 while the instance sleeps).
+// It cannot suppress other clients' activity stamps and cannot wake anything,
+// so an outsider sending it gains nothing.
+const SidecarHeader = "X-Carlos-Sidecar"
+
+// Config is the agent-process opt-in, read from env. Nil (bucket unset) keeps
+// proxy-only behavior byte-identical: instances run under systemd, the edge
+// only proxies.
+type Config struct {
+ Bucket string // CARLOS_HIBERNATE_BUCKET (required to enable)
+ Region string // CARLOS_HIBERNATE_REGION (optional; AWS default chain resolves "")
+ Prefix string // CARLOS_HIBERNATE_PREFIX (optional key prefix inside the bucket)
+ Idle time.Duration // CARLOS_HIBERNATE_IDLE (default 15m; seconds-granular, e.g. "90s")
+}
+
+// ConfigFromEnv reads the CARLOS_HIBERNATE_* env vars. A nil, nil return
+// means the bucket is unset: hibernation disabled, don't construct a Service.
+func ConfigFromEnv() (*Config, error) {
+ bucket := strings.TrimSpace(os.Getenv("CARLOS_HIBERNATE_BUCKET"))
+ if bucket == "" {
+ return nil, nil
+ }
+ cfg := &Config{
+ Bucket: bucket,
+ Region: strings.TrimSpace(os.Getenv("CARLOS_HIBERNATE_REGION")),
+ Prefix: strings.Trim(strings.TrimSpace(os.Getenv("CARLOS_HIBERNATE_PREFIX")), "/"),
+ Idle: 15 * time.Minute,
+ }
+ if v := strings.TrimSpace(os.Getenv("CARLOS_HIBERNATE_IDLE")); v != "" {
+ d, err := time.ParseDuration(v)
+ if err != nil || d <= 0 {
+ return nil, fmt.Errorf("CARLOS_HIBERNATE_IDLE %q: want a positive duration like 15m or 90s", v)
+ }
+ cfg.Idle = d
+ }
+ return cfg, nil
+}
+
+// hibernatePrefix is the per-instance S3 key prefix: <Prefix>/<host>/ (the
+// lease's lock.json is colocated under it by the lease package).
+func (c Config) hibernatePrefix(host string) string {
+ return path.Join(c.Prefix, host)
+}
+
+// mesh seam: Eleven also derived meshPrefix() here — <Prefix>/mesh/members,
+// the membership heartbeats' home beside the per-instance prefixes (the
+// bucket is the mesh's only shared substrate; instance prefixes are public
+// FQDNs and the bare label "mesh" never is one, so they can't collide).
+
+// --- backend seam ------------------------------------------------------------
+//
+// The state machine's side effects (S3 lease, restore, process spawn) sit
+// behind these interfaces so the machine itself is unit-testable with fakes —
+// wake coalescing, idle→sleep, and lease-lost stand-down need no AWS.
+
+// Backend is the side-effect seam (Eleven's actBackend). ExecBackend is the
+// production implementation.
+type Backend interface {
+ // Acquire claims the per-host single-writer lease. Contention surfaces as
+ // errLeaseHeld so the edge can answer 503 + Retry-After.
+ Acquire(ctx context.Context, host string) (Lease, error)
+ // Restore materializes the local DB from S3 when it's absent (a no-op when
+ // the warm-cache copy is still on disk).
+ Restore(ctx context.Context, host, dbPath string) error
+ // Start spawns the instance process and waits until its socket accepts. It
+ // also returns the build it execed (the live/<host> symlink target read at
+ // spawn), so the sweep can later spot a repoint under a still-running child
+ // and converge it. Empty when unknown — the sweep treats that as "don't
+ // cycle" (fail-safe).
+ Start(ctx context.Context, rt registry.Route) (Proc, string, error)
+ // NextDue asks a running instance for its earliest pending scheduled
+ // delivery (unix seconds; 0 = nothing scheduled). Queried at sleep time,
+ // while the socket still answers, to feed the due-time index.
+ NextDue(ctx context.Context, rt registry.Route) (int64, error)
+ // HasSidecar reports whether host has a sidecar staged on this box.
+ HasSidecar(host string) bool
+ // StartSidecar spawns the host's sidecar. Fire-and-forget: a sidecar is a
+ // pure outbound client with no socket, so there's nothing to health-check —
+ // its own connect/backoff loop recovers from a slow start.
+ StartSidecar(ctx context.Context, rt registry.Route) (Proc, error)
+}
+
+// DueStore persists the due-time index (Eleven's actDueStore) — the registry
+// in production (routes.next_due), so a sleeping host's timer survives agent
+// restarts instead of dying with the process. *registry.Registry satisfies it
+// directly.
+type DueStore interface {
+ SetNextDue(host string, due int64) error
+ DueInstances(now int64) ([]registry.Route, error)
+}
+
+// Lease is a held-but-not-yet-replicating lease (wake in progress; Eleven's
+// actLease).
+type Lease interface {
+ // Replicate trades the lease for running replication of the (now present)
+ // local DB; the lease handle is spent on success. syncInterval is the
+ // route's replication-class tick (replicationSyncInterval; 0 = stream).
+ Replicate(ctx context.Context, dbPath string, syncInterval time.Duration) (Rep, error)
+ // Release gives the lease up — the bail-out when a wake fails partway.
+ Release(ctx context.Context) error
+}
+
+// Rep is running replication coupled to the lease (Eleven's actRep).
+type Rep interface {
+ // Err reports lost leadership (a lease renewal failed).
+ Err() error
+ // Close does the final checkpoint/sync and releases the lease.
+ Close(ctx context.Context) error
+}
+
+// Proc is a running instance child process (Eleven's actProc).
+type Proc interface {
+ // Stop terminates gracefully (SIGTERM, bounded wait, then SIGKILL).
+ Stop(ctx context.Context) error
+ // Exited reports whether the process has died on its own.
+ Exited() bool
+}
+
+// Replication classes — the registry's routes.replication vocabulary
+// ("" | stream | batched | daily), mapped to litestream sync intervals by
+// replicationSyncInterval.
+const (
+ replicationStream = "stream" // change-driven ~1s sync — loss window seconds
+ replicationBatched = "batched" // sync at most every 5m — loss window minutes
+ replicationDaily = "daily" // sync at wake + every 24h — loss window up to a day
+)
+
+// replicationSyncInterval maps a class to the litestream replica sync
+// interval it buys. 0 means "keep litestream's default" (1s, change-driven —
+// idle ticks make zero S3 calls), which is what stream and the unset ""
+// deliberately share so pre-ladder registries keep today's behavior exactly.
+func replicationSyncInterval(class string) time.Duration {
+ switch class {
+ case replicationBatched:
+ return 5 * time.Minute
+ case replicationDaily:
+ return 24 * time.Hour
+ }
+ return 0
+}
+
+// --- state machine -----------------------------------------------------------
+
+// Per-host lifecycle. An absent entry is the fourth state: asleep.
+//
+// asleep --request--------------------> waking (one wake; concurrent
+// waking --wake ok--------------------> awake requests coalesce on it)
+// waking --wake failed----------------> asleep (error served; next request retries)
+// awake --idle timeout / lease lost /
+// child died----------------> stopping (final checkpoint, release lease)
+// stopping --stop done----------------> asleep (local DB left as a warm cache)
+//
+// A request that lands on waking waits (bounded) for the wake; one that lands
+// on stopping waits for the stop, then re-enters as a fresh wake.
+type actState int
+
+const (
+ actWaking actState = iota
+ actAwake
+ actStopping
+)
+
+type actEntry struct {
+ host string
+ route registry.Route
+ state actState
+
+ ready chan struct{} // waking: closed when the wake resolves; then check err
+ err error // waking: the wake's outcome
+ stopped chan struct{} // stopping: closed when the stop completes
+
+ // Activity tracking: an instance is idle only when nothing is in flight
+ // AND the idle window has passed since the last request finished. In-flight
+ // counting (not just timestamps) is what keeps an instance with one open
+ // WebSocket awake — the proxy's ServeHTTP blocks for the tunnel's life.
+ // (Sidecar-marked requests register in neither — see BeginSidecarRequest.)
+ inflight int
+ lastActive time.Time
+
+ rep Rep
+ proc Proc
+
+ // Convergence: the `live/<host>` symlink target this child was spawned
+ // against. A running process never re-reads the symlink, so once a ship
+ // repoints it, an awake child keeps serving the old build until it's
+ // cycled. spawnVersion lets the sweep notice the drift; staleSince is when
+ // it first did, so a brief transient (a symlink moved then rolled straight
+ // back) doesn't trigger a needless cycle. Both empty/zero until the wake
+ // resolves.
+ spawnVersion string
+ staleSince time.Time
+
+ // Sidecar co-spawn: the sidecar exists exactly while its instance is
+ // awake. All sidecar fields are written under a.mu in the awake state only
+ // (wake's attach, the sweeper's restart); a stop flips the state under the
+ // same lock first, so stop() owns them uncontended from then on.
+ sidecar Proc // the co-spawned `sidecar run` child; nil = none running
+ sidecarWanted bool // the host had a sidecar staged at wake time
+ sidecarStarted time.Time // when the current sidecar spawned (backoff reset)
+ sidecarBackoff time.Duration // last restart delay (doubles, capped)
+ sidecarRetryAt time.Time // earliest next restart attempt
+}
+
+// Sidecar restart pacing — Restart=always semantics without a supervisor: the
+// sweeper redoes a dead sidecar with exponential backoff, capped, and a run
+// that survived sidecarHealthyRun earns a fresh backoff (so one crash a day
+// never escalates to the cap).
+const (
+ sidecarRestartBase = 2 * time.Second
+ sidecarRestartMax = time.Minute
+ sidecarHealthyRun = time.Minute
+)
+
+// sidecarStopTimeout bounds the sidecar's stop at sleep time. `sidecar run`
+// has no drain handler, so the SIGTERM is effectively immediate; the SIGKILL
+// fallback behind this timeout is insurance.
+const sidecarStopTimeout = 5 * time.Second
+
+// Build convergence: how long an awake child may serve a symlink-stale binary
+// before the sweep gracefully cycles it, and how many to cycle per tick.
+// - ConvergeGrace comfortably exceeds one sweep interval (≤30s) and a
+// sidecar's ~32s reconnect, so a genuinely-persistent drift is required —
+// a symlink that moved then rolled straight back never trips it, and a
+// briefly-busy host gets a chance to idle-hibernate (which converges for
+// free) first.
+// - one per tick staggers a fleet-wide ship across sweeps instead of blipping
+// every host's connections at once.
+//
+// Convergence is a latency/observability fix, not urgent (the supply-chain bake
+// window already holds releases), so a leisurely, gentle cadence is right.
+const (
+ ConvergeGrace = 90 * time.Second
+ maxConvergeCyclesPerSweep = 1
+)
+
+// ConvergeDecision is the pure state machine behind build convergence: given
+// the symlink target read this tick (cur/readErr), the build the child execed
+// (spawnVer), and when the drift was first seen (staleSince), it says whether
+// to cycle now and returns the staleSince to carry forward. Split out so the
+// grace/reset logic is unit-testable without a live activator. Three cases:
+// - read error: unknown — neither cycle nor clear the grace; retry next tick.
+// - converged / rolled-back / nothing to compare (cur=="" or spawnVer=="" or
+// cur==spawnVer): clear the grace clock, don't cycle.
+// - stale (symlink moved since spawn): start the grace clock on first sight,
+// cycle once the drift has persisted past grace.
+func ConvergeDecision(cur string, readErr error, spawnVer string, staleSince, now time.Time, grace time.Duration) (cycle bool, nextStale time.Time) {
+ switch {
+ case readErr != nil:
+ return false, staleSince
+ case cur == "" || spawnVer == "" || cur == spawnVer:
+ return false, time.Time{}
+ default:
+ if staleSince.IsZero() {
+ staleSince = now
+ }
+ return now.Sub(staleSince) >= grace, staleSince
+ }
+}
+
+// wakeWaitBound caps how long a request queues on someone else's wake or stop
+// before giving up with 503 — a wedged wake must not pile up connections
+// forever. A var so tests can shrink it.
+var wakeWaitBound = 30 * time.Second
+
+// errLeaseHeld: the per-host lease is held elsewhere. On one box this means a
+// previous incarnation hasn't released (crashed agent; lease expires within
+// ~lease.LeaseTTL) — the edge answers 503 with Retry-After.
+type errLeaseHeld struct {
+ holder string
+ retryAfter time.Duration
+}
+
+func (e *errLeaseHeld) Error() string {
+ return fmt.Sprintf("instance lease held by %q (retry in %s)", e.holder, e.retryAfter)
+}
+
+var errWakeTimeout = errors.New("timed out waiting for instance wake")
+
+// Service is the activator (Eleven's activatorSvc). It structurally satisfies
+// the edge package's Activator interface — internal/edge is deliberately not
+// imported: the dependency points edge→activator only.
+type Service struct {
+ cfg Config
+ backend Backend
+ due DueStore
+ now func() time.Time // a seam for the unit tests' clock
+ liveVer func(string) (string, error) // live/<host> symlink target; a seam for tests
+
+ // mesh seam: Eleven kept the multi-box placement layer here (mesh *meshSvc):
+ // requests for hosts whose HRW owner is another live box were proxied
+ // there, and the due-time sweeper only woke hosts this box owned. Nil
+ // meant a single box — hashing would pick self everywhere, so no mesh
+ // codepath was touched at all. Single-box v1 drops the field; it re-enters
+ // with the mesh port.
+
+ mu sync.Mutex
+ entries map[string]*actEntry
+
+ sweepStop chan struct{}
+ sweepDone chan struct{}
+}
+
+// New wires the state machine to its side-effect backend and due-time store.
+// In production due is the *registry.Registry itself.
+func New(cfg Config, backend Backend, due DueStore) *Service {
+ a := &Service{
+ cfg: cfg,
+ backend: backend,
+ due: due,
+ now: time.Now,
+ entries: make(map[string]*actEntry),
+ }
+ // The convergence sweep's live/<host> reader. The exec backend knows the
+ // symlink layout (OptDir is its field, not a package global — Eleven used a
+ // global here); any other backend — the unit tests' fakes — leaves
+ // convergence off ("" = unknown → never cycle) unless the test wires
+ // a.liveVer itself.
+ if lv, ok := backend.(interface {
+ LiveVersion(host string) (string, error)
+ }); ok {
+ a.liveVer = lv.LiveVersion
+ } else {
+ a.liveVer = func(string) (string, error) { return "", nil }
+ }
+ return a
+}
+
+// beginRequest ensures the host's instance is awake and registers one in-flight
+// request against it. The returned release must be called when the request
+// finishes (it stamps last-activity, which starts the idle clock).
+func (a *Service) beginRequest(ctx context.Context, rt registry.Route) (release func(), err error) {
+ deadline := time.NewTimer(wakeWaitBound)
+ defer deadline.Stop()
+ for {
+ a.mu.Lock()
+ e := a.entries[rt.Host]
+ if e == nil {
+ // Asleep → kick off ONE wake; this request and any latecomers all
+ // coalesce on e.ready rather than racing a second lease claim
+ // (singleflight). The wake runs detached so even its initiator is
+ // held to the same bounded wait as everyone queued behind it.
+ e = &actEntry{host: rt.Host, route: rt, state: actWaking, ready: make(chan struct{})}
+ a.entries[rt.Host] = e
+ a.mu.Unlock()
+ go a.wake(e)
+ continue
+ }
+
+ switch e.state {
+ case actAwake:
+ // Stand down synchronously-detected failure: a dead child or a
+ // lost lease found at request time turns into a stop, and the
+ // request waits for it then re-wakes — recovery is the routine
+ // wake path.
+ if repErr := e.rep.Err(); repErr != nil || e.proc.Exited() {
+ if repErr != nil {
+ log.Printf("activator: LEASE LOST for %s (%v) — standing down", e.host, repErr)
+ } else {
+ log.Printf("activator: instance %s exited unexpectedly — recycling", e.host)
+ }
+ a.beginStopLocked(e)
+ a.mu.Unlock()
+ continue
+ }
+ e.inflight++
+ a.mu.Unlock()
+ return a.releaseFunc(e), nil
+
+ case actWaking:
+ ready := e.ready
+ a.mu.Unlock()
+ select {
+ case <-ready:
+ // A failed wake reports to every coalesced waiter (retrying
+ // here would turn one bad wake into a storm); a successful
+ // one loops to register against the now-awake entry.
+ if e.err != nil {
+ return nil, e.err
+ }
+ continue
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-deadline.C:
+ return nil, errWakeTimeout
+ }
+
+ case actStopping:
+ stopped := e.stopped
+ a.mu.Unlock()
+ select {
+ case <-stopped:
+ continue // entry is gone; loop wakes it fresh
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-deadline.C:
+ return nil, errWakeTimeout
+ }
+ }
+ }
+}
+
+// errSidecarAsleep: a sidecar-marked request arrived for a host with no awake
+// instance. Sidecar traffic never wakes anything — the activator itself spawns
+// the sidecar as part of a wake, so a marked request for a sleeping host is
+// either a stray reconnect racing a stop or an outsider spoofing the marker;
+// both get a 503 and change nothing.
+var errSidecarAsleep = errors.New("instance asleep; sidecar requests do not wake it")
+
+// BeginSidecarRequest is beginRequest for sidecar-marked traffic: pass it
+// through to an awake instance WITHOUT registering activity — no in-flight
+// count, no lastActive stamp (the sidecar's own long-lived WebSocket must not
+// hold its instance awake; that non-count is the whole reason the pair can
+// sleep) — and refuse, rather than wake, when the instance is asleep or
+// stopping.
+func (a *Service) BeginSidecarRequest(ctx context.Context, rt registry.Route) error {
+ deadline := time.NewTimer(wakeWaitBound)
+ defer deadline.Stop()
+ for {
+ a.mu.Lock()
+ e := a.entries[rt.Host]
+ if e == nil {
+ a.mu.Unlock()
+ return errSidecarAsleep
+ }
+ switch e.state {
+ case actAwake:
+ // Same dead-child / lost-lease detection as a member request, but
+ // the recovery stops at the stop: the loop then finds a stopping
+ // (or absent) entry and turns this request away instead of
+ // chaining into a re-wake.
+ if repErr := e.rep.Err(); repErr != nil || e.proc.Exited() {
+ if repErr != nil {
+ log.Printf("activator: LEASE LOST for %s (%v) — standing down", e.host, repErr)
+ } else {
+ log.Printf("activator: instance %s exited unexpectedly — recycling", e.host)
+ }
+ a.beginStopLocked(e)
+ a.mu.Unlock()
+ continue
+ }
+ a.mu.Unlock()
+ return nil
+
+ case actWaking:
+ // Joining a wake someone else started is fine — the co-spawned
+ // sidecar boots moments before its entry flips awake, so its first
+ // dial can land in that sliver; joining can't initiate anything.
+ // A failed wake just reads as asleep (the member path already
+ // logged it).
+ ready := e.ready
+ a.mu.Unlock()
+ select {
+ case <-ready:
+ if e.err != nil {
+ return errSidecarAsleep
+ }
+ continue
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-deadline.C:
+ return errWakeTimeout
+ }
+
+ default: // actStopping — the stop exists to shed exactly this connection
+ a.mu.Unlock()
+ return errSidecarAsleep
+ }
+ }
+}
+
+// mesh seam: Eleven exposed hasLiveEntry(host) here — the mesh's "already
+// living here" check (waking and stopping both counted: a waking host is
+// committed to this box, and a stopping one still holds the lease, so
+// proxying it away would only trade a short wait for a loud lease race) —
+// and awakeCount() for the fleet status page.
+
+func (a *Service) releaseFunc(e *actEntry) func() {
+ var once sync.Once
+ return func() {
+ once.Do(func() {
+ a.mu.Lock()
+ e.inflight--
+ e.lastActive = a.now()
+ a.mu.Unlock()
+ })
+ }
+}
+
+// wake drives asleep→awake: lease, restore-if-missing, spawn, replicate,
+// co-spawn the sidecar. It runs on the first requester's goroutine but under
+// its own generous timeout, detached from that request's context — queued
+// requests still need the wake to finish even if the requester who started it
+// hangs up.
+func (a *Service) wake(e *actEntry) {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+ w, err := a.doWake(ctx, e.route)
+
+ a.mu.Lock()
+ if err != nil {
+ e.err = err
+ delete(a.entries, e.host) // next request retries from scratch
+ } else {
+ e.state = actAwake
+ e.rep, e.proc = w.rep, w.proc
+ e.sidecar, e.sidecarWanted = w.sidecar, w.sidecarWanted
+ e.sidecarStarted = a.now()
+ e.sidecarBackoff = 0
+ e.sidecarRetryAt = time.Time{} // a wake-time spawn failure retries on the next sweep
+ e.lastActive = a.now()
+ // The build this child actually execed, captured at Start() inside the
+ // backend (not re-read here — a ship during the wake window would make a
+ // re-read name the new target while the process runs the old binary, and
+ // the sweep would never converge it). Empty = unknown → the sweep won't
+ // cycle. Fresh wake, so it can't be stale yet.
+ e.spawnVersion = w.spawnVersion
+ e.staleSince = time.Time{}
+ }
+ a.mu.Unlock()
+ close(e.ready)
+}
+
+// wokenInstance is a successful wake's yield. sidecar is nil when the host has
+// none staged — or when the spawn failed, which is deliberately not an error:
+// members > bots, so a host whose sidecar won't start still serves (the
+// sweeper retries via sidecarWanted).
+type wokenInstance struct {
+ rep Rep
+ proc Proc
+ sidecar Proc
+ sidecarWanted bool
+ spawnVersion string // live/<host> target read at exec — for build convergence
+}
+
+func (a *Service) doWake(ctx context.Context, rt registry.Route) (*wokenInstance, error) {
+ host := rt.Host
+ // The DB path rides the route; when empty the backend defaults it to
+ // <DataDir>/<host>.db internally (Eleven defaulted here via a package
+ // global — base dirs are backend fields now).
+ dbPath := rt.DBPath
+
+ // 1. The lease first: everything after this line is done as the prefix's
+ // single writer. Contention is not retried here — on one box it means a
+ // previous incarnation hasn't released; tell the client to come back after
+ // the TTL instead of stalling its request.
+ lease, err := a.backend.Acquire(ctx, host)
+ if err != nil {
+ var held *errLeaseHeld
+ if errors.As(err, &held) {
+ log.Printf("activator: LEASE HELD for %s by %q — a previous incarnation hasn't released; refusing to wake (retry after %s)",
+ host, held.holder, held.retryAfter)
+ }
+ return nil, err
+ }
+
+ // 2. Restore the DB from S3 when the local copy is gone — the restore must
+ // happen *under* the lock (restoring without it could later shadow newer
+ // S3 state written by the true holder). When it's still on disk it is the
+ // warm cache: on one box, with the lease held, the local copy can only be
+ // what we ourselves checkpointed at the last sleep.
+ if err := a.backend.Restore(ctx, host, dbPath); err != nil {
+ _ = lease.Release(ctx)
+ return nil, fmt.Errorf("restore %s: %w", host, err)
+ }
+
+ // 3. Spawn before replication starts: a fresh instance creates its own DB
+ // on boot, so waiting for the socket guarantees the file exists for
+ // litestream to open. Writes that land in the gap are picked up by the
+ // replica's first sync.
+ proc, spawnVer, err := a.backend.Start(ctx, rt)
+ if err != nil {
+ _ = lease.Release(ctx)
+ return nil, fmt.Errorf("start %s: %w", host, err)
+ }
+
+ // 4. Trade the held lease for running replication, at the route's
+ // replication-class sync interval (the per-plan durability ladder).
+ rep, err := lease.Replicate(ctx, dbPath, replicationSyncInterval(rt.Replication))
+ if err != nil {
+ stopCtx, cancel := context.WithTimeout(context.Background(), actStopTimeout)
+ defer cancel()
+ _ = proc.Stop(stopCtx)
+ _ = lease.Release(ctx)
+ return nil, fmt.Errorf("replicate %s: %w", host, err)
+ }
+
+ // 5. Co-spawn the sidecar, last — only once the instance is healthy and
+ // replicating, so the sidecar always boots against a live instance (its
+ // marked requests pass through the edge while we're awake). A spawn
+ // failure is loud but never fails the wake: members come before bots, and
+ // the sweeper retries with backoff.
+ w := &wokenInstance{rep: rep, proc: proc, spawnVersion: spawnVer}
+ if a.backend.HasSidecar(host) {
+ w.sidecarWanted = true
+ if sc, err := a.backend.StartSidecar(ctx, rt); err != nil {
+ log.Printf("activator: SIDECAR for %s failed to start: %v — serving without it (the sweep retries)", host, err)
+ } else {
+ w.sidecar = sc
+ }
+ }
+ return w, nil
+}
+
+// actStopTimeout bounds a graceful instance stop (the instance's own SIGTERM
+// drain is expected ~10s; leave headroom before the SIGKILL).
+const actStopTimeout = 20 * time.Second
+
+// beginStopLocked flips an awake entry to stopping and kicks the stop off; the
+// caller holds a.mu. In-flight requests are not waited for — a stop is only
+// initiated when the instance is idle, dead, or has lost its lease, and in the
+// latter two cases open requests are already doomed.
+func (a *Service) beginStopLocked(e *actEntry) {
+ e.state = actStopping
+ e.stopped = make(chan struct{})
+ go a.stop(e)
+}
+
+// stop drives awake→asleep: stop the child (its SIGTERM drain is the final
+// write), checkpoint what's on disk to S3 and release the lease, then forget
+// the entry. The local DB stays in place — it's the warm cache for the next
+// wake; S3 is the durable truth under the lease.
+func (a *Service) stop(e *actEntry) {
+ // Co-stop, sidecar FIRST — before the next-due probe and the instance's
+ // own SIGTERM. Two reasons: the sidecar must never outlive its instance
+ // (an orphan's reconnect loop would be a standing wake source — the
+ // ping-pong this feature exists to prevent — and its scheduled work would
+ // keep firing into a host we're about to stop), and with the sidecar gone
+ // first, the schedule the next-due probe reads can't shift under it.
+ if e.sidecar != nil {
+ scCtx, scCancel := context.WithTimeout(context.Background(), sidecarStopTimeout)
+ if err := e.sidecar.Stop(scCtx); err != nil {
+ log.Printf("activator: stop sidecar %s: %v", e.host, err)
+ }
+ scCancel()
+ }
+
+ // The due-time index: ask the instance for its earliest pending scheduled
+ // delivery BEFORE stopping it — the socket dies with the process. (When the
+ // stop was triggered by a dead child, the query just fails fast.) Any
+ // failure records 0, i.e. "sleeps until a request": a missed probe must
+ // degrade to pre-index behavior, never block or crash the sleep. A due time
+ // already in the past is recorded as-is and the sweeper wakes the host on
+ // its next tick — there is deliberately no minimum-sleep guard beyond that,
+ // because thrash is already impossible: an awake instance only sleeps after
+ // a full idle window, so wake/sleep cycles can never be spaced closer than
+ // cfg.Idle.
+ dueCtx, cancelDue := context.WithTimeout(context.Background(), 5*time.Second)
+ due, err := a.backend.NextDue(dueCtx, e.route)
+ cancelDue()
+ if err != nil {
+ log.Printf("activator: next-due probe %s: %v — recording none (it sleeps until a request)", e.host, err)
+ due = 0
+ }
+ if err := a.due.SetNextDue(e.host, due); err != nil {
+ log.Printf("activator: record next-due %s: %v", e.host, err)
+ } else if due > 0 {
+ log.Printf("activator: %s next scheduled delivery at %s — will wake for it", e.host, time.Unix(due, 0).UTC().Format(time.RFC3339))
+ }
+
+ // Separate budgets: a SIGTERM drain that runs long must not eat the final
+ // checkpoint's time — the S3 sync is the part that makes sleep durable.
+ procCtx, cancelProc := context.WithTimeout(context.Background(), actStopTimeout)
+ err = e.proc.Stop(procCtx)
+ cancelProc()
+ if err != nil {
+ log.Printf("activator: stop %s: %v", e.host, err)
+ }
+ // Close = final sync + lease release. After a lost lease the release part
+ // fails harmlessly (someone else owns the lock); the sync still preserves
+ // a local checkpoint.
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ if err := e.rep.Close(ctx); err != nil {
+ log.Printf("activator: final checkpoint %s: %v", e.host, err)
+ }
+ a.mu.Lock()
+ delete(a.entries, e.host)
+ a.mu.Unlock()
+ close(e.stopped)
+ log.Printf("activator: %s hibernated", e.host)
+}
+
+// Start launches the background sweeper that puts idle instances to sleep and
+// stands down lost leases/dead children between requests.
+func (a *Service) Start() {
+ a.sweepStop = make(chan struct{})
+ a.sweepDone = make(chan struct{})
+ // Sweep well inside the idle window so a sleep happens promptly after it
+ // elapses; clamp so second-granular test idles still sweep sanely.
+ interval := a.cfg.Idle / 4
+ if interval < 100*time.Millisecond {
+ interval = 100 * time.Millisecond
+ }
+ if interval > 30*time.Second {
+ interval = 30 * time.Second
+ }
+ go func() {
+ defer close(a.sweepDone)
+ t := time.NewTicker(interval)
+ defer t.Stop()
+ for {
+ select {
+ case <-a.sweepStop:
+ return
+ case <-t.C:
+ a.sweepOnce()
+ }
+ }
+ }()
+}
+
+// sweepOnce examines every awake instance once: idle ones hibernate; a lost
+// lease or dead child stands down immediately regardless of activity. It then
+// checks the due-time index and wakes any sleeping host whose scheduled
+// delivery has come due.
+func (a *Service) sweepOnce() {
+ now := a.now()
+ // The due query runs before the lock — it's a registry read, and nothing
+ // under a.mu should wait on SQLite.
+ dueRoutes, err := a.due.DueInstances(now.Unix())
+ if err != nil {
+ log.Printf("activator: due-time sweep: %v", err)
+ }
+ a.mu.Lock()
+ cycled := 0 // build-convergence cycles started this tick (rate-limited)
+ for _, e := range a.entries {
+ if e.state != actAwake {
+ continue
+ }
+ if err := e.rep.Err(); err != nil {
+ log.Printf("activator: LEASE LOST for %s (%v) — standing down", e.host, err)
+ a.beginStopLocked(e)
+ continue
+ }
+ if e.proc.Exited() {
+ log.Printf("activator: instance %s exited unexpectedly — recycling", e.host)
+ a.beginStopLocked(e)
+ continue
+ }
+ if e.inflight == 0 && now.Sub(e.lastActive) >= a.cfg.Idle {
+ log.Printf("activator: %s idle %s — hibernating", e.host, now.Sub(e.lastActive).Truncate(time.Second))
+ a.beginStopLocked(e)
+ continue
+ }
+ // Converge onto a shipped build. A running child never re-reads
+ // live/<host>, so once a ship repoints the symlink this awake child is
+ // serving a stale binary. Idle hosts converge for free on their next
+ // wake; this cycle is for the always-awake ones — notably every
+ // sidecar host, which never sleeps — that otherwise serve the old
+ // build indefinitely. Gracefully cycle: beginStopLocked drains via
+ // SIGTERM, takes the final checkpoint, releases the lease, and leaves
+ // the host asleep; a member request (or the due-time index) then wakes
+ // it onto the new binary — the same clean cycle `systemctl restart`
+ // gives a unit-managed host. Unlike idle hibernation this can fire
+ // with a request in flight; the stop still drains HTTP gracefully and
+ // reconnecting clients coalesce onto the fresh wake, so a long upload
+ // past the drain is cut exactly as a restart would cut it. A pinned
+ // canary never trips this: the updater leaves its symlink alone, so
+ // spawnVersion still matches.
+ cur, readErr := a.liveVer(e.host)
+ cycle, next := ConvergeDecision(cur, readErr, e.spawnVersion, e.staleSince, now, ConvergeGrace)
+ e.staleSince = next
+ if cycle && cycled < maxConvergeCyclesPerSweep {
+ log.Printf("activator: %s serving %s but live symlink is now %s — cycling onto the new build", e.host, e.spawnVersion, cur)
+ a.beginStopLocked(e)
+ cycled++
+ continue
+ }
+ // Sidecar supervision — Restart=always, sweep-driven, no supervisor
+ // goroutine: a sidecar that died (or failed to spawn at wake) while
+ // its instance stays awake is respawned with doubling, capped backoff;
+ // a run past sidecarHealthyRun resets it. Spawned under a.mu
+ // deliberately: an exec is quick, and the lock is what makes a restart
+ // impossible to race with a stop (sidecar fields only change in the
+ // awake state, and every stop flips the state under this same lock
+ // first).
+ //
+ // A sidecar env staged MID-FLIGHT (attached to an awake child — the
+ // opt-in path) is ADOPTED here rather than waiting for the next wake,
+ // so opting in takes effect in seconds. Safe against double-running:
+ // only hibernating hosts are activator children and their systemd
+ // sidecar units stay disabled — the staged sidecar has exactly one
+ // possible runner.
+ if !e.sidecarWanted && a.backend.HasSidecar(e.host) {
+ e.sidecarWanted = true
+ log.Printf("activator: sidecar staged for awake %s — adopting", e.host)
+ }
+ if e.sidecarWanted && (e.sidecar == nil || e.sidecar.Exited()) && !now.Before(e.sidecarRetryAt) {
+ if e.sidecar != nil && now.Sub(e.sidecarStarted) >= sidecarHealthyRun {
+ e.sidecarBackoff = 0
+ }
+ if e.sidecarBackoff == 0 {
+ e.sidecarBackoff = sidecarRestartBase
+ } else {
+ e.sidecarBackoff = min(e.sidecarBackoff*2, sidecarRestartMax)
+ }
+ e.sidecarRetryAt = now.Add(e.sidecarBackoff)
+ sc, err := a.backend.StartSidecar(context.Background(), e.route)
+ if err != nil {
+ log.Printf("activator: SIDECAR restart for %s failed: %v (next attempt in %s)", e.host, err, e.sidecarBackoff)
+ e.sidecar = nil
+ continue
+ }
+ log.Printf("activator: sidecar for %s was down — restarted (backoff now %s)", e.host, e.sidecarBackoff)
+ e.sidecar = sc
+ e.sidecarStarted = now
+ }
+ }
+ // Due-time wakes: a request-less wake through the same singleflight state
+ // machine, so a client request arriving mid-wake just coalesces. Only hosts
+ // with NO live entry are woken — an awake host is already serving its
+ // schedule; a waking one is on its way; a stopping one will re-record its
+ // next_due when the stop completes and be picked up on a later tick. The
+ // registry's stale past value while the host is up is harmless for the same
+ // reason. A failed due wake (e.g. a lease still held by a crashed
+ // incarnation) removes the entry, and the index — only rewritten at the
+ // next successful sleep — naturally retries it every tick until it works.
+ for _, rt := range dueRoutes {
+ // Only hibernating, ready hosts: the index exists to wake sleeping
+ // instances, and a systemd-managed host is always on — spawning a
+ // child for it would fight its unit over the socket. This also makes
+ // a leftover next_due inert on a host flipped back to systemd (the
+ // flip-off's release re-records one after the flag is already clear).
+ // The registry's DueInstances is deliberately broader (no hibernate
+ // filter), so the gate lives here, with the machine it protects.
+ if !rt.Hibernate || rt.Status != "ready" {
+ continue
+ }
+ if _, live := a.entries[rt.Host]; live {
+ continue
+ }
+ // mesh seam: in the multi-box mesh only the HRW owner due-wakes a host —
+ // otherwise every box sweeping a due entry would race a wake per tick
+ // (the lease would let exactly one win, but loudly, forever). Eleven:
+ // `if a.mesh != nil && !a.mesh.owns(rt.Host) { continue }`.
+ log.Printf("activator: %s has a scheduled delivery due — waking it", rt.Host)
+ e := &actEntry{host: rt.Host, route: rt, state: actWaking, ready: make(chan struct{})}
+ a.entries[rt.Host] = e
+ go a.wake(e)
+ }
+ a.mu.Unlock()
+}
+
+// RetireExcept stops (final checkpoint + lease release) every live entry whose
+// host is not in keep — the reload-time reconcile behind flipping a host's
+// hibernate flag off (and, as a bonus, route removal while awake). The flip
+// clears the route's flag and reloads the edge; the map swap stops routing
+// requests through the activator, and this puts the child down and gives the
+// S3 lease back so the host's systemd unit can bind the socket. It reuses the
+// ordinary stop path — the narrowest hook that exists — so the release still
+// records next_due and checkpoints to S3 like any sleep. In-flight requests on
+// a retired child are cut off as a deploy restart would cut them — the flip is
+// a deliberate operator action.
+func (a *Service) RetireExcept(keep map[string]bool) {
+ a.mu.Lock()
+ var waking []*actEntry
+ for _, e := range a.entries {
+ if keep[e.host] {
+ continue
+ }
+ switch e.state {
+ case actAwake:
+ log.Printf("activator: %s is no longer hibernate-managed — releasing", e.host)
+ a.beginStopLocked(e)
+ case actWaking:
+ // A wake can't be stopped mid-flight; finish the retire once it
+ // resolves (below, off-lock).
+ waking = append(waking, e)
+ case actStopping:
+ // Already on its way out.
+ }
+ }
+ a.mu.Unlock()
+ for _, e := range waking {
+ go func(e *actEntry) {
+ <-e.ready
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ // Only if THIS entry is still the live one and its wake succeeded —
+ // a failed wake already removed itself, and a fresh entry for the
+ // same host means the route was re-flagged in the meantime.
+ if a.entries[e.host] == e && e.state == actAwake {
+ log.Printf("activator: %s is no longer hibernate-managed — releasing (wake finished first)", e.host)
+ a.beginStopLocked(e)
+ }
+ }(e)
+ }
+}
+
+// Shutdown checkpoints and stops every child — the agent going down takes its
+// instances with it (they're its children; leaving them orphaned would strand
+// the leases until TTL expiry and leave the sockets bound against the next
+// incarnation's spawns). Blocks until all stops complete or ctx expires.
+func (a *Service) Shutdown(ctx context.Context) {
+ // mesh seam — drain step 1, order-pinned: Eleven left the mesh membership
+ // FIRST — the heartbeat is deleted before any checkpoint or lease release
+ // below, so peers stop proxying here immediately and their wakes succeed
+ // the moment each lease releases. Deploys, box failure drills, and
+ // scale-down all ride this same path. (`a.mesh.stop(ctx)` went here.)
+ if a.sweepStop != nil {
+ close(a.sweepStop)
+ <-a.sweepDone
+ }
+ a.mu.Lock()
+ var waits []chan struct{}
+ for _, e := range a.entries {
+ switch e.state {
+ case actAwake:
+ a.beginStopLocked(e)
+ waits = append(waits, e.stopped)
+ case actStopping:
+ waits = append(waits, e.stopped)
+ case actWaking:
+ // A mid-flight wake resolves on its own; its instance will be
+ // orphaned only if it wins the race with process exit — acceptable
+ // for the rare shutdown-during-wake, and the lease TTL still
+ // bounds the damage.
+ }
+ }
+ a.mu.Unlock()
+ for _, w := range waits {
+ select {
+ case <-w:
+ case <-ctx.Done():
+ return
+ }
+ }
+}
+
+// ServeHTTP answers one proxied request for a hibernating instance: wake (or
+// join a wake), proxy, stamp activity. Lease contention and wake timeouts are
+// 503s — "sleeping hosts are slow to wake", never a 5xx cascade of retries.
+// It reports whether it handled the request (wrote the response or proxied
+// it); false means "not activator-managed — proxy straight", so the edge can
+// call it unconditionally.
+func (a *Service) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
+ // Per-host gating: only a hibernate-flagged instance is activator-managed;
+ // unflagged instances and sibling services are the edge's to proxy
+ // straight — the deploy-safety property that lets the bucket env ship
+ // fleet-wide as a no-op before any host is flagged.
+ if rt.Kind != "instance" || !rt.Hibernate {
+ return false
+ }
+
+ // Sidecar-marked traffic (SidecarHeader — the sidecar stamps every REST
+ // call and its WebSocket dial) is activity-invisible: proxied while the
+ // instance is awake, refused while it isn't, never a wake. The co-spawned
+ // sidecar dials its instance's public URL through this very edge (the box
+ // pins its hosted names to loopback), so without this its keepalives
+ // would read as member activity — the instance could never idle — and
+ // after a sleep its reconnect loop would be the wake source: ping-pong
+ // forever. Spoofing the marker is harmless by construction: it only
+ // DOWNGRADES the sender's own requests (no activity credit; 503 while
+ // asleep). It cannot suppress other clients' activity stamps and cannot
+ // wake a sleeping host, so an outsider sending it gains nothing.
+ if r.Header.Get(SidecarHeader) != "" {
+ // The marker is an edge-side signal only (activity exclusion, above);
+ // strip it so nothing downstream can ever grow a meaning an outsider
+ // could spoof.
+ r.Header.Del(SidecarHeader)
+ if err := a.BeginSidecarRequest(r.Context(), rt); err != nil {
+ if errors.Is(err, context.Canceled) {
+ return true // the sidecar hung up while queued on a wake
+ }
+ w.Header().Set("Retry-After", "5")
+ http.Error(w, "instance is asleep; sidecar requests do not wake it", http.StatusServiceUnavailable)
+ return true
+ }
+ proxy.ServeHTTP(w, r)
+ return true
+ }
+
+ // mesh seam: Eleven proxied a member request for a host with no live
+ // local entry and an HRW owner on another live box to that peer here (one
+ // hop max, guarded by a forwarded marker); a host with ANY live local
+ // entry stayed local — placement converges at the next sleep boundary,
+ // since its state is already in S3. Single-box v1 always wakes locally.
+
+ release, err := a.beginRequest(r.Context(), rt)
+ if err != nil {
+ var held *errLeaseHeld
+ switch {
+ case errors.As(err, &held):
+ retry := int(held.retryAfter/time.Second) + 1
+ w.Header().Set("Retry-After", strconv.Itoa(retry))
+ http.Error(w, "instance is waking elsewhere; retry shortly", http.StatusServiceUnavailable)
+ case errors.Is(err, errWakeTimeout):
+ w.Header().Set("Retry-After", "5")
+ http.Error(w, "instance is waking; retry shortly", http.StatusServiceUnavailable)
+ case errors.Is(err, context.Canceled):
+ // The client hung up while queued; nothing to write.
+ default:
+ log.Printf("activator: wake %s: %v", rt.Host, err)
+ http.Error(w, "instance failed to wake", http.StatusServiceUnavailable)
+ }
+ return true
+ }
+ defer release()
+ proxy.ServeHTTP(w, r)
+ return true
+}
diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go
new file mode 100644
index 000000000..c3970f40c
--- /dev/null
+++ b/internal/activator/activator_test.go
@@ -0,0 +1,1841 @@
+package activator
+
+// Ported from lchat activator_test.go.
+//
+// Unit tests for the activator state machine (activator.go) with a fake
+// backend — wake coalescing, idle→sleep, lease-lost stand-down, contention
+// 503s. No AWS, no processes. The router-level tests use a small in-package
+// test router that mimics the edge's dispatch (internal/edge) — deliberately
+// NOT imported: the activator satisfies edge.Activator structurally, and the
+// dependency must point edge→activator only.
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/http/httputil"
+ "os"
+ "path/filepath"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// --- fakes -------------------------------------------------------------------
+
+type fakeRep struct {
+ mu sync.Mutex
+ err error
+ closed bool
+}
+
+func (r *fakeRep) Err() error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return r.err
+}
+func (r *fakeRep) setErr(err error) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.err = err
+}
+func (r *fakeRep) Close(context.Context) error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.closed = true
+ return nil
+}
+func (r *fakeRep) isClosed() bool {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return r.closed
+}
+
+type fakeProc struct {
+ mu sync.Mutex
+ stopped bool
+ dead bool
+ // stopGate, when non-nil, blocks Stop() until closed — holds an entry in
+ // the stopping state so a test can poke at it.
+ stopGate chan struct{}
+ // label + rec feed the backend's order log ("instance" / "sidecar"), so
+ // the co-spawn/co-stop tests can assert interleavings.
+ label string
+ rec *fakeBackend
+}
+
+func (p *fakeProc) Stop(context.Context) error {
+ p.mu.Lock()
+ gate := p.stopGate
+ p.mu.Unlock()
+ if gate != nil {
+ <-gate
+ }
+ p.mu.Lock()
+ p.stopped = true
+ p.mu.Unlock()
+ if p.rec != nil {
+ p.rec.record("stop:" + p.label)
+ }
+ return nil
+}
+func (p *fakeProc) Exited() bool {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return p.dead
+}
+func (p *fakeProc) isStopped() bool {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return p.stopped
+}
+
+type fakeLease struct {
+ rep *fakeRep
+ b *fakeBackend // records the sync interval the wake threads through
+}
+
+func (l *fakeLease) Replicate(_ context.Context, _ string, syncInterval time.Duration) (Rep, error) {
+ if l.b != nil {
+ l.b.mu.Lock()
+ l.b.syncIntervals = append(l.b.syncIntervals, syncInterval)
+ l.b.mu.Unlock()
+ }
+ return l.rep, nil
+}
+func (l *fakeLease) Release(context.Context) error { return nil }
+
+type fakeBackend struct {
+ mu sync.Mutex
+ acquires int
+ restores int
+ starts int
+ acquireErr error
+ startGate chan struct{} // when non-nil, Start() blocks until closed
+ reps []*fakeRep
+ procs []*fakeProc
+ // syncIntervals records, per wake, the replication-class tick doWake
+ // threads into Replicate() — the ladder's seam assertion.
+ syncIntervals []time.Duration
+ dueVal int64 // what the instance reports at the sleep-time next-due probe
+ dueErr error // when non-nil, the probe fails (dead socket, missing token, …)
+
+ // execVerFn models the live/<host> target the child execs at spawn — Start()
+ // returns it as the spawnVersion, exactly as ExecBackend reads the symlink
+ // just before Start(). Nil → "" (unknown), which disables convergence, so
+ // tests that don't care are unaffected. Point it at the same source as the
+ // sweep's liveVer seam to model a symlink faithfully (both read the current
+ // target).
+ execVerFn func() string
+
+ // Sidecar co-spawn. sidecarStaged is HasSidecar's answer (false by
+ // default, so every pre-co-spawn test doubles as the "host without a
+ // sidecar spawns none" case); sidecarErr injects spawn failures.
+ sidecarStaged bool
+ sidecarErr error
+ sidecarStarts int
+ sidecars []*fakeProc
+ // order interleaves backend calls and proc stops (start:instance,
+ // start:sidecar, nextDue, stop:sidecar, stop:instance) for the ordering
+ // assertions.
+ order []string
+}
+
+func (b *fakeBackend) record(ev string) {
+ b.mu.Lock()
+ b.order = append(b.order, ev)
+ b.mu.Unlock()
+}
+
+func (b *fakeBackend) events() []string {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return append([]string(nil), b.order...)
+}
+
+func (b *fakeBackend) Acquire(context.Context, string) (Lease, error) {
+ b.mu.Lock()
+ b.acquires++
+ err := b.acquireErr
+ rep := &fakeRep{}
+ b.reps = append(b.reps, rep)
+ b.mu.Unlock()
+ if err != nil {
+ return nil, err
+ }
+ return &fakeLease{rep: rep, b: b}, nil
+}
+
+func (b *fakeBackend) Restore(context.Context, string, string) error {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ b.restores++
+ return nil
+}
+
+func (b *fakeBackend) Start(context.Context, registry.Route) (Proc, string, error) {
+ b.mu.Lock()
+ b.starts++
+ b.order = append(b.order, "start:instance")
+ gate := b.startGate
+ fn := b.execVerFn
+ proc := &fakeProc{label: "instance", rec: b}
+ b.procs = append(b.procs, proc)
+ b.mu.Unlock()
+ if gate != nil {
+ <-gate
+ }
+ ver := ""
+ if fn != nil {
+ ver = fn()
+ }
+ return proc, ver, nil
+}
+
+func (b *fakeBackend) NextDue(context.Context, registry.Route) (int64, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ b.order = append(b.order, "nextDue")
+ if b.dueErr != nil {
+ return 0, b.dueErr
+ }
+ return b.dueVal, nil
+}
+
+func (b *fakeBackend) HasSidecar(string) bool {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.sidecarStaged
+}
+
+func (b *fakeBackend) StartSidecar(context.Context, registry.Route) (Proc, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ b.sidecarStarts++
+ b.order = append(b.order, "start:sidecar")
+ if b.sidecarErr != nil {
+ return nil, b.sidecarErr
+ }
+ p := &fakeProc{label: "sidecar", rec: b}
+ b.sidecars = append(b.sidecars, p)
+ return p, nil
+}
+
+func (b *fakeBackend) sidecarStartCount() int {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.sidecarStarts
+}
+
+func (b *fakeBackend) lastSidecar() *fakeProc {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.sidecars[len(b.sidecars)-1]
+}
+
+func (b *fakeBackend) setSidecarErr(err error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ b.sidecarErr = err
+}
+
+func (b *fakeBackend) setDue(v int64) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ b.dueVal = v
+}
+
+func (b *fakeBackend) counts() (acquires, restores, starts int) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.acquires, b.restores, b.starts
+}
+
+func (b *fakeBackend) lastProc() *fakeProc {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.procs[len(b.procs)-1]
+}
+
+func (b *fakeBackend) lastRep() *fakeRep {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.reps[len(b.reps)-1]
+}
+
+// fakeClock is the activator's `now` seam.
+type fakeClock struct {
+ mu sync.Mutex
+ t time.Time
+}
+
+func (c *fakeClock) now() time.Time {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.t
+}
+func (c *fakeClock) advance(d time.Duration) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.t = c.t.Add(d)
+}
+
+// The registry IS the DueStore in production — assert the wiring compiles.
+var _ DueStore = (*registry.Registry)(nil)
+
+// testActivator wires the fakes to a REAL (temp-dir) registry as the due store,
+// so the due-time tests exercise the actual routes.next_due SQL, not a fake of it.
+func testActivator(t *testing.T, idle time.Duration) (*Service, *fakeBackend, *fakeClock, *registry.Registry) {
+ t.Helper()
+ b := &fakeBackend{}
+ clk := &fakeClock{t: time.Unix(1_700_000_000, 0)}
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatalf("open registry: %v", err)
+ }
+ a := New(Config{Bucket: "test-bucket", Idle: idle}, b, reg)
+ a.now = clk.now
+ return a, b, clk, reg
+}
+
+func testRoute(host string) registry.Route {
+ // Hibernate is set because these routes model activator-managed hosts —
+ // the per-host gate (the due sweep's flag filter, ServeHTTP's gate) only
+ // acts on flagged instances.
+ return registry.Route{Host: host, Kind: "instance", Socket: "/tmp/nope.sock",
+ DBPath: filepath.Join("/tmp", host+".db"), Hibernate: true}
+}
+
+// waitFor polls until a condition holds — the stop path runs on its own
+// goroutine, so tests observe its completion rather than assuming it.
+func waitFor(t *testing.T, what string, cond func() bool) {
+ t.Helper()
+ deadline := time.Now().Add(5 * time.Second)
+ for time.Now().Before(deadline) {
+ if cond() {
+ return
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ t.Fatalf("timed out waiting for %s", what)
+}
+
+func (a *Service) entryCount() int {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ return len(a.entries)
+}
+
+// --- the test router ---------------------------------------------------------
+//
+// A minimal stand-in for internal/edge's dispatch: routes with a socket get a
+// unix reverse proxy; hibernate-flagged instance routes go through the
+// activator's ServeHTTP hook first (the same shape as edge.Edge.ServeHTTP);
+// reload() re-lists the registry and calls RetireExcept with the managed set,
+// exactly as edge.Reload does.
+
+type testUpstream struct {
+ proxy http.Handler
+ route registry.Route
+ hib bool
+}
+
+type testRouter struct {
+ reg *registry.Registry
+ act *Service
+ mu sync.Mutex
+ up map[string]testUpstream
+}
+
+func newTestRouter(reg *registry.Registry, act *Service) (*testRouter, error) {
+ rt := &testRouter{reg: reg, act: act}
+ if err := rt.reload(); err != nil {
+ return nil, err
+ }
+ return rt, nil
+}
+
+func (rt *testRouter) reload() error {
+ routes, err := rt.reg.List()
+ if err != nil {
+ return err
+ }
+ m := make(map[string]testUpstream)
+ managed := make(map[string]bool)
+ for _, t := range routes {
+ hib := rt.act != nil && t.Kind == "instance" && t.Hibernate
+ if hib {
+ managed[t.Host] = true
+ }
+ m[t.Host] = testUpstream{proxy: testUnixProxy(t.Socket), route: t, hib: hib}
+ }
+ rt.mu.Lock()
+ rt.up = m
+ rt.mu.Unlock()
+ if rt.act != nil {
+ rt.act.RetireExcept(managed)
+ }
+ return nil
+}
+
+func (rt *testRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ host := r.Host
+ if h, _, err := net.SplitHostPort(host); err == nil {
+ host = h
+ }
+ rt.mu.Lock()
+ up, ok := rt.up[host]
+ rt.mu.Unlock()
+ if !ok {
+ http.Error(w, "unknown host", http.StatusNotFound)
+ return
+ }
+ if up.hib && rt.act.ServeHTTP(w, r, up.route, up.proxy) {
+ return
+ }
+ up.proxy.ServeHTTP(w, r)
+}
+
+func testUnixProxy(socket string) http.Handler {
+ dialer := &net.Dialer{}
+ return &httputil.ReverseProxy{
+ Rewrite: func(pr *httputil.ProxyRequest) {
+ pr.Out.URL.Scheme = "http"
+ pr.Out.URL.Host = "carlos" // placeholder; the unix dialer ignores it
+ pr.Out.Host = pr.In.Host
+ },
+ Transport: &http.Transport{
+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+ return dialer.DialContext(ctx, "unix", socket)
+ },
+ },
+ }
+}
+
+// proxyGet sends GET base+"/" with an overridden Host header and returns the
+// response body + status code.
+func proxyGet(t *testing.T, base, host string) (string, int) {
+ t.Helper()
+ req, err := http.NewRequest(http.MethodGet, base+"/", nil)
+ if err != nil {
+ t.Fatalf("new request: %v", err)
+ }
+ req.Host = host
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("request: %v", err)
+ }
+ defer resp.Body.Close()
+ b, _ := io.ReadAll(resp.Body)
+ return string(b), resp.StatusCode
+}
+
+// tmpSock returns a short unix socket path (the sun_path limit is ~104 bytes,
+// so t.TempDir() can be too long) and schedules its removal.
+func tmpSock(t *testing.T) string {
+ t.Helper()
+ s := filepath.Join(os.TempDir(), fmt.Sprintf("carlos-act-%d-%d.sock", os.Getpid(), time.Now().UnixNano()%1_000_000))
+ os.Remove(s)
+ t.Cleanup(func() { os.Remove(s) })
+ return s
+}
+
+// --- tests ---------------------------------------------------------------------
+
+// Many concurrent requests during a wake coalesce into exactly one wake: one
+// lease claim, one restore, one spawn.
+func TestActivatorWakeCoalesces(t *testing.T) {
+ a, b, _, _ := testActivator(t, time.Minute)
+ b.startGate = make(chan struct{})
+
+ const n = 12
+ var ok atomic.Int32
+ var wg sync.WaitGroup
+ for i := 0; i < n; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Errorf("beginRequest: %v", err)
+ return
+ }
+ ok.Add(1)
+ release()
+ }()
+ }
+ // Let the requests pile up on the gated wake, then open it.
+ waitFor(t, "the wake to begin", func() bool { _, _, s := b.counts(); return s == 1 })
+ time.Sleep(50 * time.Millisecond)
+ close(b.startGate)
+ wg.Wait()
+
+ if got := ok.Load(); got != n {
+ t.Fatalf("%d/%d requests succeeded", got, n)
+ }
+ acquires, restores, starts := b.counts()
+ if acquires != 1 || restores != 1 || starts != 1 {
+ t.Fatalf("wake did not coalesce: acquires=%d restores=%d starts=%d (want 1 each)", acquires, restores, starts)
+ }
+}
+
+// The wake threads the route's replication class into Replicate() as the
+// litestream sync interval — the per-plan durability ladder's seam. Unset
+// (every pre-ladder route) must thread 0, i.e. litestream's default: a deploy
+// of the ladder changes no host's durability by itself.
+func TestActivatorWakeThreadsReplicationClass(t *testing.T) {
+ a, b, _, _ := testActivator(t, time.Minute)
+
+ for i, tc := range []struct {
+ class string
+ want time.Duration
+ }{
+ {"", 0},
+ {replicationStream, 0},
+ {replicationBatched, 5 * time.Minute},
+ {replicationDaily, 24 * time.Hour},
+ } {
+ rt := testRoute(fmt.Sprintf("ladder%d.example.com", i))
+ rt.Replication = tc.class
+ release, err := a.beginRequest(context.Background(), rt)
+ if err != nil {
+ t.Fatalf("beginRequest(%q): %v", tc.class, err)
+ }
+ release()
+ b.mu.Lock()
+ got := b.syncIntervals[len(b.syncIntervals)-1]
+ b.mu.Unlock()
+ if got != tc.want {
+ t.Errorf("class %q threaded sync interval %v, want %v", tc.class, got, tc.want)
+ }
+ }
+}
+
+// After the idle window with nothing in flight, the sweeper hibernates the
+// instance: process stopped, replication closed (final checkpoint + lease
+// release), entry forgotten; the next request wakes it afresh.
+func TestActivatorIdleSleep(t *testing.T) {
+ a, b, clk, _ := testActivator(t, 10*time.Second)
+
+ release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+
+ // Not yet idle: nothing happens.
+ clk.advance(9 * time.Second)
+ a.sweepOnce()
+ if a.entryCount() != 1 {
+ t.Fatal("instance hibernated before its idle window elapsed")
+ }
+
+ clk.advance(2 * time.Second)
+ a.sweepOnce()
+ waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
+ if !b.lastProc().isStopped() {
+ t.Fatal("idle instance's process was not stopped")
+ }
+ if !b.lastRep().isClosed() {
+ t.Fatal("idle instance's replication was not closed (no final checkpoint/lease release)")
+ }
+
+ // A new request wakes it again — a second full wake.
+ release, err = a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest after sleep: %v", err)
+ }
+ release()
+ if _, _, starts := b.counts(); starts != 2 {
+ t.Fatalf("starts=%d after sleep+rewake, want 2", starts)
+ }
+}
+
+// An in-flight request (an open WebSocket, in production) holds the instance
+// awake past the idle window; only after it finishes does the idle clock run.
+func TestActivatorInflightKeepsAwake(t *testing.T) {
+ a, b, clk, _ := testActivator(t, 10*time.Second)
+
+ release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+
+ clk.advance(time.Hour)
+ a.sweepOnce()
+ if a.entryCount() != 1 || b.lastProc().isStopped() {
+ t.Fatal("instance with an in-flight request was hibernated")
+ }
+
+ release() // stamps lastActive = now
+ a.sweepOnce()
+ if a.entryCount() != 1 {
+ t.Fatal("instance hibernated immediately after its last request finished")
+ }
+ clk.advance(11 * time.Second)
+ a.sweepOnce()
+ waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
+}
+
+// A ship repoints live/<host> under an awake child. A running process never
+// re-reads its symlink, so idle-hibernate is the only thing that converges an
+// instance — and an always-awake host (every sidecar host) never idles.
+// The sweep closes that gap: it notices the drift and, after a grace window and
+// rate-limited to one per tick, gracefully cycles the child so its next wake
+// execs the new binary.
+func TestActivatorConvergesOnSymlinkRepoint(t *testing.T) {
+ a, b, clk, _ := testActivator(t, time.Hour) // idle far away — isolate the convergence path
+ var ver atomic.Value // the live/<host> symlink target; swapped to model a ship
+ ver.Store("v1")
+ // Model one symlink: the child execs whatever it points at now (execVerFn,
+ // frozen into spawnVersion at wake), and the sweep reads the current target
+ // (liveVer). A "ship" swaps ver; the running child keeps its frozen version.
+ a.liveVer = func(string) (string, error) { return ver.Load().(string), nil }
+ b.execVerFn = func() string { return ver.Load().(string) }
+
+ release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+
+ // Symlink unchanged: never cycles, however long it stays awake.
+ clk.advance(3 * ConvergeGrace)
+ a.sweepOnce()
+ if a.entryCount() != 1 {
+ t.Fatal("cycled an instance whose symlink never moved")
+ }
+
+ // Ship: repoint the symlink. The first sweep after only starts the grace
+ // clock — it must not cycle yet, so a move-then-rollback never drops a live
+ // connection.
+ ver.Store("v2")
+ a.sweepOnce()
+ if a.entryCount() != 1 {
+ t.Fatal("cycled inside the grace window — a transient repoint would drop a live connection")
+ }
+
+ // Past the grace: gracefully cycle (stop → final checkpoint → lease release →
+ // asleep). The reconnecting client/sidecar would wake it onto v2.
+ clk.advance(ConvergeGrace + time.Second)
+ a.sweepOnce()
+ waitFor(t, "convergence cycle", func() bool { return a.entryCount() == 0 })
+ if !b.lastProc().isStopped() {
+ t.Fatal("stale instance's process was not stopped")
+ }
+ if !b.lastRep().isClosed() {
+ t.Fatal("convergence cycle skipped the final checkpoint / lease release")
+ }
+
+ // Re-wake against the now-current symlink: spawnVersion tracks v2, so it
+ // never cycles again — no thrash.
+ release, err = a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("re-wake after convergence: %v", err)
+ }
+ release()
+ clk.advance(3 * ConvergeGrace)
+ a.sweepOnce()
+ if a.entryCount() != 1 {
+ t.Fatal("cycled again after already converging onto the current build")
+ }
+}
+
+// Two awake children go stale at once (a fleet-wide ship). One sweep cycles at
+// most one — a staggered convergence, never a simultaneous blip of every space.
+func TestActivatorConvergenceRateLimitsPerSweep(t *testing.T) {
+ a, b, clk, _ := testActivator(t, time.Hour)
+ var ver atomic.Value
+ ver.Store("v1")
+ a.liveVer = func(string) (string, error) { return ver.Load().(string), nil }
+ b.execVerFn = func() string { return ver.Load().(string) }
+
+ for _, h := range []string{"one.example.com", "two.example.com"} {
+ release, err := a.beginRequest(context.Background(), testRoute(h))
+ if err != nil {
+ t.Fatalf("beginRequest %s: %v", h, err)
+ }
+ release()
+ }
+ if a.entryCount() != 2 {
+ t.Fatalf("want 2 awake instances, got %d", a.entryCount())
+ }
+
+ ver.Store("v2") // ship under both
+ a.sweepOnce() // starts the grace clock for both
+ clk.advance(ConvergeGrace + time.Second)
+
+ a.sweepOnce() // both are past grace, but only one may cycle this tick
+ waitFor(t, "first convergence cycle", func() bool { return a.entryCount() == 1 })
+ // A moment later, the sweep takes the second.
+ a.sweepOnce()
+ waitFor(t, "second convergence cycle", func() bool { return a.entryCount() == 0 })
+}
+
+// The feature's whole premise: converge a host that NEVER reaches inflight==0
+// (an open WebSocket, or a sidecar's persistent connection), which
+// idle-hibernate can never touch. Unlike idle-hibernate, convergence fires with
+// a request in flight — the graceful stop drains it exactly as a restart would.
+func TestActivatorConvergesWithRequestInFlight(t *testing.T) {
+ a, b, clk, _ := testActivator(t, time.Hour)
+ var ver atomic.Value
+ ver.Store("v1")
+ a.liveVer = func(string) (string, error) { return ver.Load().(string), nil }
+ b.execVerFn = func() string { return ver.Load().(string) }
+
+ // Hold the request open for the whole test — inflight stays 1, so the idle
+ // path (which needs inflight==0) can never converge this host.
+ release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ defer release()
+
+ ver.Store("v2") // ship under the busy host
+ a.sweepOnce() // grace starts; must not cycle a busy host mid-grace
+ if a.entryCount() != 1 {
+ t.Fatal("cycled inside the grace window")
+ }
+ clk.advance(ConvergeGrace + time.Second)
+ a.sweepOnce()
+ // Cycles despite the in-flight request — idle-hibernate never would.
+ waitFor(t, "busy-host convergence", func() bool { return a.entryCount() == 0 })
+ if !b.lastProc().isStopped() {
+ t.Fatal("busy stale instance's process was not stopped")
+ }
+}
+
+// ConvergeDecision's state machine, exhaustively: the grace clock starts on
+// first drift, survives a rollback correctly, and a read error never clears
+// accrued grace (it means "unknown", not "converged").
+func TestConvergeDecision(t *testing.T) {
+ base := time.Unix(1_700_000_000, 0)
+ set := base.Add(-40 * time.Second) // a grace clock that started 40s ago
+ errBoom := errors.New("readlink boom")
+ cases := []struct {
+ name string
+ cur string
+ readErr error
+ spawnVer string
+ staleSince time.Time
+ wantCycle bool
+ wantStale time.Time
+ }{
+ {"matches — clears grace", "v1", nil, "v1", set, false, time.Time{}},
+ {"empty cur — no data, clears", "", nil, "v1", set, false, time.Time{}},
+ {"empty spawnVer — unknown, clears", "v2", nil, "", set, false, time.Time{}},
+ {"first drift — starts grace, no cycle", "v2", nil, "v1", time.Time{}, false, base},
+ {"drift within grace — holds, no cycle", "v2", nil, "v1", base.Add(-30 * time.Second), false, base.Add(-30 * time.Second)},
+ {"drift past grace — cycles", "v2", nil, "v1", base.Add(-ConvergeGrace - time.Second), true, base.Add(-ConvergeGrace - time.Second)},
+ {"read error — preserves accrued grace", "", errBoom, "v1", set, false, set},
+ {"read error before any drift — stays zero", "", errBoom, "v1", time.Time{}, false, time.Time{}},
+ {"rollback to spawn — clears grace", "v1", nil, "v1", base.Add(-ConvergeGrace - time.Second), false, time.Time{}},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ cycle, next := ConvergeDecision(c.cur, c.readErr, c.spawnVer, c.staleSince, base, ConvergeGrace)
+ if cycle != c.wantCycle {
+ t.Errorf("cycle = %v, want %v", cycle, c.wantCycle)
+ }
+ if !next.Equal(c.wantStale) {
+ t.Errorf("nextStale = %v, want %v", next, c.wantStale)
+ }
+ })
+ }
+}
+
+// A lost lease (failed renewal) stands the instance down immediately — no
+// waiting for idle: someone else may own the S3 prefix now.
+func TestActivatorLeaseLostStandsDown(t *testing.T) {
+ a, b, _, _ := testActivator(t, time.Hour)
+
+ release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+
+ b.lastRep().setErr(errors.New("lease renewal failed"))
+ a.sweepOnce() // idle window is an hour away; the lost lease alone must trigger the stop
+ waitFor(t, "stand-down", func() bool { return a.entryCount() == 0 })
+ if !b.lastProc().isStopped() {
+ t.Fatal("lease-lost instance's process was not stopped")
+ }
+ if !b.lastRep().isClosed() {
+ t.Fatal("lease-lost instance's replication was not closed")
+ }
+}
+
+// A dead child (crashed instance) is recycled by the next request: the stale
+// entry is stopped and a fresh wake happens in the same beginRequest call.
+func TestActivatorDeadChildRewakesOnRequest(t *testing.T) {
+ a, b, _, _ := testActivator(t, time.Hour)
+
+ release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+
+ p := b.lastProc()
+ p.mu.Lock()
+ p.dead = true
+ p.mu.Unlock()
+
+ release, err = a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest after crash: %v", err)
+ }
+ release()
+ if _, _, starts := b.counts(); starts != 2 {
+ t.Fatalf("starts=%d after crash+request, want 2 (recycle + rewake)", starts)
+ }
+}
+
+// A request that arrives mid-stop waits for the stop, then wakes fresh.
+func TestActivatorRequestDuringStopRewakes(t *testing.T) {
+ a, b, clk, _ := testActivator(t, 10*time.Second)
+
+ release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+
+ // Hold the stop open so the entry sits in stopping.
+ gate := make(chan struct{})
+ p := b.lastProc()
+ p.mu.Lock()
+ p.stopGate = gate
+ p.mu.Unlock()
+
+ clk.advance(time.Minute)
+ a.sweepOnce()
+
+ got := make(chan error, 1)
+ go func() {
+ release, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if err == nil {
+ release()
+ }
+ got <- err
+ }()
+ time.Sleep(50 * time.Millisecond) // the request should now be parked on e.stopped
+ close(gate)
+ if err := <-got; err != nil {
+ t.Fatalf("request during stop: %v", err)
+ }
+ if _, _, starts := b.counts(); starts != 2 {
+ t.Fatalf("starts=%d, want 2 (sleep then rewake)", starts)
+ }
+}
+
+// A wake that exceeds the bound turns queued requests away with errWakeTimeout
+// (503 at the edge) rather than piling connections up behind it.
+func TestActivatorWakeTimeout(t *testing.T) {
+ a, b, _, _ := testActivator(t, time.Minute)
+ old := wakeWaitBound
+ wakeWaitBound = 100 * time.Millisecond
+ t.Cleanup(func() { wakeWaitBound = old })
+
+ gate := make(chan struct{})
+ b.startGate = gate
+ t.Cleanup(func() { close(gate) }) // let the stuck wake finish at test end
+
+ _, err := a.beginRequest(context.Background(), testRoute("one.example.com"))
+ if !errors.Is(err, errWakeTimeout) {
+ t.Fatalf("got %v, want errWakeTimeout", err)
+ }
+}
+
+// Lease contention surfaces through the edge as 503 + Retry-After, and a
+// failed wake leaves no entry behind (the next request retries the claim).
+func TestActivatorLeaseContention503(t *testing.T) {
+ a, b, _, reg := testActivator(t, time.Minute)
+ b.acquireErr = &errLeaseHeld{holder: "old-incarnation", retryAfter: 7 * time.Second}
+
+ if err := reg.Upsert(registry.Route{Host: "held.example.com", Kind: "instance", Socket: "/tmp/held.sock", Hibernate: true}); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+ rt, err := newTestRouter(reg, a)
+ if err != nil {
+ t.Fatalf("new router: %v", err)
+ }
+ front := httptest.NewServer(rt)
+ t.Cleanup(front.Close)
+
+ req, _ := http.NewRequest("GET", front.URL+"/", nil)
+ req.Host = "held.example.com"
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("request: %v", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusServiceUnavailable {
+ t.Fatalf("status %d, want 503", resp.StatusCode)
+ }
+ if ra := resp.Header.Get("Retry-After"); ra != "8" { // 7s truncated + 1
+ t.Fatalf("Retry-After %q, want 8", ra)
+ }
+ if a.entryCount() != 0 {
+ t.Fatal("failed wake left an entry behind")
+ }
+}
+
+// Per-host gating (the opt-in): with the activator CONFIGURED, only a
+// hibernate-flagged instance goes through the wake path; an unflagged one is
+// proxied straight, exactly as if there were no activator at all — the
+// deploy-safety property that lets the bucket env ship fleet-wide before any
+// host is flagged.
+func TestRouterPerHostHibernateGating(t *testing.T) {
+ a, b, _, reg := testActivator(t, time.Minute)
+
+ // Two live backends playing the instances. The flagged host's listener
+ // stands in for the child a wake would have spawned (the fake backend's
+ // Start() spawns nothing).
+ bind := func(name string) string {
+ sock := tmpSock(t)
+ ln, err := net.Listen("unix", sock)
+ if err != nil {
+ t.Fatalf("listen unix: %v", err)
+ }
+ srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ io.WriteString(w, name)
+ })}
+ go srv.Serve(ln)
+ t.Cleanup(func() { srv.Close() })
+ return sock
+ }
+ if err := reg.Upsert(registry.Route{Host: "hib.example.com", Kind: "instance", Socket: bind("hib"), Hibernate: true}); err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.Upsert(registry.Route{Host: "plain.example.com", Kind: "instance", Socket: bind("plain")}); err != nil {
+ t.Fatal(err)
+ }
+
+ rt, err := newTestRouter(reg, a)
+ if err != nil {
+ t.Fatalf("new router: %v", err)
+ }
+ front := httptest.NewServer(rt)
+ t.Cleanup(front.Close)
+
+ // The unflagged host: proxied straight, the activator never touched.
+ if body, code := proxyGet(t, front.URL, "plain.example.com"); code != 200 || body != "plain" {
+ t.Fatalf("unflagged host: got %d %q, want 200 plain", code, body)
+ }
+ if acquires, _, starts := b.counts(); acquires != 0 || starts != 0 {
+ t.Fatalf("unflagged instance touched the activator (acquires=%d starts=%d)", acquires, starts)
+ }
+
+ // The flagged host: served through a full wake — lease, restore, spawn.
+ if body, code := proxyGet(t, front.URL, "hib.example.com"); code != 200 || body != "hib" {
+ t.Fatalf("flagged host: got %d %q, want 200 hib", code, body)
+ }
+ if acquires, restores, starts := b.counts(); acquires != 1 || restores != 1 || starts != 1 {
+ t.Fatalf("flagged host skipped the wake path: acquires=%d restores=%d starts=%d (want 1 each)", acquires, restores, starts)
+ }
+}
+
+// The activator's own gate: handed an unflagged (or non-instance) route it
+// declines — returns false, writes nothing, touches no backend — so the edge
+// proxies straight. This is the bool half of the edge.Activator contract.
+func TestServeHTTPUnmanagedRouteReturnsFalse(t *testing.T) {
+ a, b, _, _ := testActivator(t, time.Minute)
+ noop := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest("GET", "http://plain.example.com/", nil)
+ unflagged := registry.Route{Host: "plain.example.com", Kind: "instance", Socket: "/tmp/x.sock"}
+ if a.ServeHTTP(rec, req, unflagged, noop) {
+ t.Fatal("unflagged instance route was handled by the activator")
+ }
+ svc := registry.Route{Host: "svc.example.com", Kind: "service", Socket: "/tmp/s.sock", Hibernate: true}
+ if a.ServeHTTP(rec, req, svc, noop) {
+ t.Fatal("service route was handled by the activator")
+ }
+ if rec.Body.Len() != 0 {
+ t.Fatalf("declined routes wrote a response: %q", rec.Body.String())
+ }
+ if acquires, _, starts := b.counts(); acquires != 0 || starts != 0 {
+ t.Fatalf("declined routes touched the backend (acquires=%d starts=%d)", acquires, starts)
+ }
+}
+
+// A reload that finds an awake host no longer hibernate-flagged retires its
+// child — stopped, final-checkpointed (replication closed), entry forgotten —
+// the edge-process half of flipping hibernation off. And once the flag is
+// off, a recorded next_due no longer wakes it: the due index only acts on
+// hibernating hosts, so a systemd-managed instance can never be spawned
+// against its own unit.
+func TestActivatorReloadRetiresUnflaggedHost(t *testing.T) {
+ a, b, clk, reg := testActivator(t, time.Hour)
+ route := testRoute("flip.example.com")
+ if err := reg.Upsert(route); err != nil {
+ t.Fatal(err)
+ }
+ rt, err := newTestRouter(reg, a)
+ if err != nil {
+ t.Fatalf("new router: %v", err)
+ }
+
+ release, err := a.beginRequest(context.Background(), route)
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+ // The retire's sleep re-records next_due AFTER the flag is cleared; an
+ // already-past value must stay inert under the flag filter.
+ b.setDue(clk.now().Add(-5 * time.Second).Unix())
+
+ if err := reg.SetHibernate("flip.example.com", false); err != nil {
+ t.Fatal(err)
+ }
+ if err := rt.reload(); err != nil {
+ t.Fatalf("reload: %v", err)
+ }
+ waitFor(t, "retire", func() bool { return a.entryCount() == 0 })
+ if !b.lastProc().isStopped() {
+ t.Fatal("retired child was not stopped")
+ }
+ if !b.lastRep().isClosed() {
+ t.Fatal("retired child's replication was not closed (no final checkpoint/lease release)")
+ }
+
+ // The due sweeper must not resurrect a host that's back under systemd.
+ a.sweepOnce()
+ if _, _, starts := b.counts(); starts != 1 {
+ t.Fatalf("starts=%d after retire, want 1 (no due wake for a systemd-managed host)", starts)
+ }
+}
+
+// A retire that lands mid-wake can't stop the child yet — it finishes the job
+// once the wake resolves, so a flip-off during the small wake window still
+// releases cleanly.
+func TestActivatorRetireDuringWake(t *testing.T) {
+ a, b, clk, reg := testActivator(t, time.Hour)
+ route := testRoute("mid.example.com")
+ if err := reg.Upsert(route); err != nil {
+ t.Fatal(err)
+ }
+ rt, err := newTestRouter(reg, a)
+ if err != nil {
+ t.Fatalf("new router: %v", err)
+ }
+
+ // Start a request-less (due-time) wake and hold it in the waking state.
+ b.startGate = make(chan struct{})
+ if err := reg.SetNextDue("mid.example.com", clk.now().Add(-time.Second).Unix()); err != nil {
+ t.Fatal(err)
+ }
+ a.sweepOnce()
+ waitFor(t, "the wake to begin", func() bool { _, _, s := b.counts(); return s == 1 })
+
+ // Flip off mid-wake: the reload's retire must defer, not deadlock.
+ if err := reg.SetHibernate("mid.example.com", false); err != nil {
+ t.Fatal(err)
+ }
+ if err := rt.reload(); err != nil {
+ t.Fatalf("reload: %v", err)
+ }
+ close(b.startGate)
+
+ waitFor(t, "post-wake retire", func() bool { return a.entryCount() == 0 })
+ if !b.lastProc().isStopped() {
+ t.Fatal("child woken mid-retire was not stopped")
+ }
+ if !b.lastRep().isClosed() {
+ t.Fatal("child woken mid-retire left replication open (lease never released)")
+ }
+}
+
+// With hibernation on, sibling services (kind "service") are untouched: the
+// edge proxies them directly, no wake, no activity tracking.
+func TestActivatorServiceRoutesBypass(t *testing.T) {
+ sock := tmpSock(t)
+ ln, err := net.Listen("unix", sock)
+ if err != nil {
+ t.Fatalf("listen unix: %v", err)
+ }
+ backend := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ io.WriteString(w, "service ok")
+ })}
+ go backend.Serve(ln)
+ t.Cleanup(func() { backend.Close() })
+
+ a, b, _, _ := testActivator(t, time.Minute)
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatalf("open registry: %v", err)
+ }
+ if err := reg.Upsert(registry.Route{Host: "svc.example.com", Kind: "service", Socket: sock}); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+ rt, err := newTestRouter(reg, a)
+ if err != nil {
+ t.Fatalf("new router: %v", err)
+ }
+ front := httptest.NewServer(rt)
+ t.Cleanup(front.Close)
+
+ req, _ := http.NewRequest("GET", front.URL+"/", nil)
+ req.Host = "svc.example.com"
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("request: %v", err)
+ }
+ body, _ := io.ReadAll(resp.Body)
+ resp.Body.Close()
+ if resp.StatusCode != 200 || string(body) != "service ok" {
+ t.Fatalf("got %d %q, want 200 service ok", resp.StatusCode, body)
+ }
+ if acquires, _, _ := b.counts(); acquires != 0 {
+ t.Fatal("a service route touched the activator")
+ }
+}
+
+// Shutdown checkpoints and stops every awake child before the edge exits.
+func TestActivatorShutdownStopsChildren(t *testing.T) {
+ a, b, _, _ := testActivator(t, time.Hour)
+ for _, h := range []string{"one.example.com", "two.example.com"} {
+ release, err := a.beginRequest(context.Background(), testRoute(h))
+ if err != nil {
+ t.Fatalf("beginRequest %s: %v", h, err)
+ }
+ release()
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ a.Shutdown(ctx)
+
+ if a.entryCount() != 0 {
+ t.Fatal("shutdown left entries behind")
+ }
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ for i, p := range b.procs {
+ if !p.stopped {
+ t.Fatalf("proc %d not stopped on shutdown", i)
+ }
+ }
+ for i, r := range b.reps {
+ if !r.closed {
+ t.Fatalf("replication %d not closed on shutdown", i)
+ }
+ }
+}
+
+// --- due-time index --------------------------------------------------------------
+
+// entryAwake reports whether host has a live entry in the awake state — how the
+// due tests observe an async due-time wake resolving.
+func (a *Service) entryAwake(host string) bool {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ e := a.entries[host]
+ return e != nil && e.state == actAwake
+}
+
+// The full self-maintaining cycle: sleep records the instance's next due time
+// in the registry; the sweeper wakes the host (no request) when it arrives; the
+// following sleep re-records — and once nothing is scheduled, no more due wakes.
+func TestActivatorDueTimeWakeCycle(t *testing.T) {
+ a, b, clk, reg := testActivator(t, 10*time.Second)
+ route := testRoute("due.example.com")
+ if err := reg.Upsert(route); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+
+ // Wake by request; the instance has a delivery scheduled 30s out.
+ dueAt := clk.now().Add(30 * time.Second).Unix()
+ b.setDue(dueAt)
+ release, err := a.beginRequest(context.Background(), route)
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+
+ // Idle out: the sleep probes the instance and records its due time.
+ clk.advance(11 * time.Second)
+ a.sweepOnce()
+ waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
+ if rt, err := reg.Get(route.Host); err != nil || rt == nil || rt.NextDue != dueAt {
+ t.Fatalf("registry next_due = %+v, %v; want %d", rt, err, dueAt)
+ }
+
+ // Before the due time: sweeps leave it asleep.
+ a.sweepOnce()
+ if _, _, starts := b.counts(); starts != 1 {
+ t.Fatalf("woke before its due time (starts=%d)", starts)
+ }
+
+ // At the due time: the sweeper wakes it with no client request.
+ clk.advance(20 * time.Second) // now = wake+31s ≥ due
+ a.sweepOnce()
+ waitFor(t, "due-time wake", func() bool { return a.entryAwake(route.Host) })
+ if _, _, starts := b.counts(); starts != 2 {
+ t.Fatalf("starts=%d after due wake, want 2", starts)
+ }
+
+ // The woken instance stays up for the normal idle window, then sleeps and
+ // re-records — nothing scheduled now, so the index clears and stays quiet.
+ b.setDue(0)
+ clk.advance(11 * time.Second)
+ a.sweepOnce()
+ waitFor(t, "re-hibernation", func() bool { return a.entryCount() == 0 })
+ if rt, _ := reg.Get(route.Host); rt.NextDue != 0 {
+ t.Fatalf("next_due = %d after empty-schedule sleep, want 0", rt.NextDue)
+ }
+ clk.advance(time.Hour)
+ a.sweepOnce()
+ if _, _, starts := b.counts(); starts != 2 {
+ t.Fatalf("starts=%d with nothing scheduled, want 2 (no phantom due wake)", starts)
+ }
+}
+
+// A due time already in the past at sleep time (a delivery that raced the stop)
+// is recorded as-is: the sweeper wakes the host on its next tick — once — and
+// there is no ping-pong, because a woken instance can't sleep again until a
+// full idle window passes.
+func TestActivatorDuePastWakesOnceNoPingPong(t *testing.T) {
+ a, b, clk, reg := testActivator(t, 10*time.Second)
+ route := testRoute("late.example.com")
+ if err := reg.Upsert(route); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+ b.setDue(clk.now().Add(-5 * time.Second).Unix()) // overdue before it even sleeps
+
+ release, err := a.beginRequest(context.Background(), route)
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+ clk.advance(11 * time.Second)
+ a.sweepOnce()
+ waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
+
+ // Next tick: woken for the overdue delivery.
+ a.sweepOnce()
+ waitFor(t, "overdue wake", func() bool { return a.entryAwake(route.Host) })
+
+ // No thrash: with the wake fresh (clock unmoved), repeated sweeps neither
+ // re-sleep it nor double-wake it, even though next_due is still in the past.
+ for i := 0; i < 5; i++ {
+ a.sweepOnce()
+ }
+ if a.entryCount() != 1 || !a.entryAwake(route.Host) {
+ t.Fatal("due-woken instance did not stay awake through its idle window")
+ }
+ if _, _, starts := b.counts(); starts != 2 {
+ t.Fatalf("starts=%d, want 2 (request wake + one due wake)", starts)
+ }
+}
+
+// A failed next-due probe (dead socket, missing admin token, …) records 0 and
+// the sleep proceeds — the instance degrades to "sleeps until a request",
+// never a crash and never a stuck stop.
+func TestActivatorDueProbeFailureRecordsZero(t *testing.T) {
+ a, b, clk, reg := testActivator(t, 10*time.Second)
+ route := testRoute("deaf.example.com")
+ if err := reg.Upsert(route); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+ b.dueErr = errors.New("dial unix: no such file")
+
+ release, err := a.beginRequest(context.Background(), route)
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+ clk.advance(11 * time.Second)
+ a.sweepOnce()
+ waitFor(t, "hibernation despite probe failure", func() bool { return a.entryCount() == 0 })
+
+ if rt, _ := reg.Get(route.Host); rt.NextDue != 0 {
+ t.Fatalf("next_due = %d after failed probe, want 0", rt.NextDue)
+ }
+ clk.advance(time.Hour)
+ a.sweepOnce()
+ if _, _, starts := b.counts(); starts != 1 {
+ t.Fatalf("starts=%d, want 1 (a failed probe must not invent a wake)", starts)
+ }
+}
+
+// The index lives in the registry, so a due time recorded before an edge
+// restart still fires: a brand-new activator over the same registry — no
+// in-memory state carried over — wakes the host when its time arrives.
+func TestActivatorDueSurvivesRestart(t *testing.T) {
+ a1, b1, clk, reg := testActivator(t, 10*time.Second)
+ route := testRoute("reborn.example.com")
+ if err := reg.Upsert(route); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+ b1.setDue(clk.now().Add(30 * time.Second).Unix())
+
+ release, err := a1.beginRequest(context.Background(), route)
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+ clk.advance(11 * time.Second)
+ a1.sweepOnce()
+ waitFor(t, "hibernation", func() bool { return a1.entryCount() == 0 })
+
+ // "Restart": a fresh activator, same registry.
+ b2 := &fakeBackend{}
+ a2 := New(Config{Bucket: "test-bucket", Idle: 10 * time.Second}, b2, reg)
+ a2.now = clk.now
+
+ clk.advance(25 * time.Second) // past the due time
+ a2.sweepOnce()
+ waitFor(t, "post-restart due wake", func() bool { return a2.entryAwake(route.Host) })
+ if _, _, starts := b2.counts(); starts != 1 {
+ t.Fatalf("starts=%d on the new activator, want 1", starts)
+ }
+}
+
+// --- sidecar co-spawn ----------------------------------------------------------
+
+// indexOf: position of ev in the backend's order log (-1 = absent).
+func indexOf(events []string, ev string) int {
+ for i, e := range events {
+ if e == ev {
+ return i
+ }
+ }
+ return -1
+}
+
+// A host with a sidecar staged gets it co-spawned on wake, strictly AFTER the
+// instance process is up — the sidecar dials the instance, so spawning it
+// first would just burn its reconnect backoff. (Every other test in this file
+// runs with sidecarStaged=false and asserts nothing sidecar-ish happens, which
+// is the "host without a sidecar behaves exactly as today" half.)
+func TestActivatorCoSpawnsSidecarAfterInstance(t *testing.T) {
+ a, b, _, _ := testActivator(t, time.Minute)
+ b.sidecarStaged = true
+
+ release, err := a.beginRequest(context.Background(), testRoute("bots.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+
+ if got := b.sidecarStartCount(); got != 1 {
+ t.Fatalf("sidecar starts = %d, want 1", got)
+ }
+ ev := b.events()
+ if i, s := indexOf(ev, "start:instance"), indexOf(ev, "start:sidecar"); i == -1 || s == -1 || s < i {
+ t.Fatalf("sidecar did not start strictly after the instance: %v", ev)
+ }
+}
+
+// A sidecar that fails to spawn must not fail the wake — members > bots: the
+// host serves without its sidecar, and the sweeper keeps retrying with backoff
+// until a spawn sticks.
+func TestActivatorSidecarFailureDoesNotFailWake(t *testing.T) {
+ a, b, clk, _ := testActivator(t, time.Hour)
+ b.sidecarStaged = true
+ b.setSidecarErr(errors.New("exec: live symlink gone"))
+ route := testRoute("botless.example.com")
+
+ release, err := a.beginRequest(context.Background(), route)
+ if err != nil {
+ t.Fatalf("beginRequest with a failing sidecar: %v", err)
+ }
+ release()
+ if !a.entryAwake(route.Host) {
+ t.Fatal("instance not awake after a sidecar-only failure")
+ }
+ if got := b.sidecarStartCount(); got != 1 {
+ t.Fatalf("sidecar attempts = %d, want 1 (the wake's)", got)
+ }
+
+ // The sweeper retries: attempt 2 immediately (first retry), then gated by
+ // the doubling backoff; once the fault clears, a retry lands the sidecar.
+ a.sweepOnce()
+ if got := b.sidecarStartCount(); got != 2 {
+ t.Fatalf("sidecar attempts after first sweep = %d, want 2", got)
+ }
+ a.sweepOnce() // still inside the 2s gate — no attempt
+ if got := b.sidecarStartCount(); got != 2 {
+ t.Fatalf("backoff gate ignored: attempts = %d, want 2", got)
+ }
+ b.setSidecarErr(nil)
+ clk.advance(3 * time.Second)
+ a.sweepOnce()
+ if got := b.sidecarStartCount(); got != 3 {
+ t.Fatalf("sidecar attempts after fault cleared = %d, want 3", got)
+ }
+ if b.lastSidecar().isStopped() || b.lastSidecar().Exited() {
+ t.Fatal("recovered sidecar isn't running")
+ }
+}
+
+// Co-stop ordering at sleep: the sidecar is stopped FIRST — before the
+// next-due probe (so bots can't shift the schedule being recorded) and before
+// the instance's own stop (so the sidecar never outlives its instance).
+func TestActivatorCoStopOrdering(t *testing.T) {
+ a, b, clk, _ := testActivator(t, 10*time.Second)
+ b.sidecarStaged = true
+
+ release, err := a.beginRequest(context.Background(), testRoute("bots.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+
+ clk.advance(11 * time.Second)
+ a.sweepOnce()
+ waitFor(t, "hibernation", func() bool { return a.entryCount() == 0 })
+
+ if !b.lastSidecar().isStopped() {
+ t.Fatal("sidecar not stopped at sleep")
+ }
+ ev := b.events()
+ sc, due, inst := indexOf(ev, "stop:sidecar"), indexOf(ev, "nextDue"), indexOf(ev, "stop:instance")
+ if sc == -1 || due == -1 || inst == -1 || !(sc < due && due < inst) {
+ t.Fatalf("co-stop order wrong: %v (want stop:sidecar < nextDue < stop:instance)", ev)
+ }
+}
+
+// Orphan prevention: whatever takes the instance down — here a crashed child,
+// but a lost lease and a retire ride the identical stop path — takes the
+// sidecar with it. An orphaned sidecar would reconnect-loop against the edge
+// forever (503s at best; a wake source if the marker rule ever broke).
+func TestActivatorInstanceDeathStopsSidecar(t *testing.T) {
+ a, b, _, _ := testActivator(t, time.Hour)
+ b.sidecarStaged = true
+
+ release, err := a.beginRequest(context.Background(), testRoute("bots.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+
+ p := b.lastProc()
+ p.mu.Lock()
+ p.dead = true
+ p.mu.Unlock()
+ a.sweepOnce()
+ waitFor(t, "stand-down", func() bool { return a.entryCount() == 0 })
+ if !b.lastSidecar().isStopped() {
+ t.Fatal("instance died but its sidecar was left running (orphan)")
+ }
+}
+
+// Sidecar death while the instance stays awake: restarted with doubling,
+// capped backoff — Restart=always semantics, sweep-driven — and a run that
+// held for sidecarHealthyRun earns a fresh backoff.
+func TestActivatorSidecarRestartBackoff(t *testing.T) {
+ a, b, clk, _ := testActivator(t, time.Hour)
+ b.sidecarStaged = true
+
+ release, err := a.beginRequest(context.Background(), testRoute("bots.example.com"))
+ if err != nil {
+ t.Fatalf("beginRequest: %v", err)
+ }
+ release()
+ kill := func() {
+ sc := b.lastSidecar()
+ sc.mu.Lock()
+ sc.dead = true
+ sc.mu.Unlock()
+ }
+
+ // T0: first crash — restarted on the next sweep (attempt 2), gate 2s.
+ kill()
+ a.sweepOnce()
+ if got := b.sidecarStartCount(); got != 2 {
+ t.Fatalf("first restart: attempts = %d, want 2", got)
+ }
+ // Crash-loop: still inside the 2s gate — no restart.
+ kill()
+ a.sweepOnce()
+ if got := b.sidecarStartCount(); got != 2 {
+ t.Fatalf("gate ignored: attempts = %d, want 2", got)
+ }
+ // T3: past the 2s gate — attempt 3, gate doubles to 4s.
+ clk.advance(3 * time.Second)
+ a.sweepOnce()
+ if got := b.sidecarStartCount(); got != 3 {
+ t.Fatalf("second restart: attempts = %d, want 3", got)
+ }
+ // T6: 3s later is inside the 4s gate — the doubling is real.
+ kill()
+ clk.advance(3 * time.Second)
+ a.sweepOnce()
+ if got := b.sidecarStartCount(); got != 3 {
+ t.Fatalf("doubled gate ignored: attempts = %d, want 3", got)
+ }
+ // T8: past it — attempt 4 (gate now 8s).
+ clk.advance(2 * time.Second)
+ a.sweepOnce()
+ if got := b.sidecarStartCount(); got != 4 {
+ t.Fatalf("third restart: attempts = %d, want 4", got)
+ }
+ // A healthy run (≥ sidecarHealthyRun) resets the ladder: the next crash
+ // restarts immediately and its gate is back to the 2s base…
+ clk.advance(2 * time.Minute)
+ kill()
+ a.sweepOnce()
+ if got := b.sidecarStartCount(); got != 5 {
+ t.Fatalf("post-healthy restart: attempts = %d, want 5", got)
+ }
+ // …which a 3s advance clears (16s — the unreset ladder — would not).
+ kill()
+ clk.advance(3 * time.Second)
+ a.sweepOnce()
+ if got := b.sidecarStartCount(); got != 6 {
+ t.Fatalf("backoff not reset after a healthy run: attempts = %d, want 6", got)
+ }
+}
+
+// The activity question: the co-spawned sidecar dials its instance's public
+// URL through this very edge (the box pins its hosted names to loopback), so
+// without the marker its keepalives would be member activity and the pair
+// could never sleep. Full front-door check: marked requests are served while
+// awake — with the marker stripped before the backend — but stamp nothing
+// (the host hibernates on schedule beneath them), and once asleep they get
+// 503 and wake nothing. An outsider spoofing the marker gets exactly that
+// same treatment: it can't wake the host and can't suppress anyone else's
+// stamps (stamps are per-request), so the marker is not a DoS lever in either
+// direction.
+func TestActivatorSidecarRequestsInvisibleToActivity(t *testing.T) {
+ a, b, clk, reg := testActivator(t, 10*time.Second)
+
+ // A live backend listener stands in for the child a wake would have
+ // spawned (the fake backend's Start() spawns nothing), recording whether
+ // the sidecar marker ever leaks through the proxy.
+ sock := tmpSock(t)
+ ln, err := net.Listen("unix", sock)
+ if err != nil {
+ t.Fatalf("listen unix: %v", err)
+ }
+ var leaked atomic.Bool
+ srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get(SidecarHeader) != "" {
+ leaked.Store(true)
+ }
+ io.WriteString(w, "ok")
+ })}
+ go srv.Serve(ln)
+ t.Cleanup(func() { srv.Close() })
+
+ const host = "bots.example.com"
+ if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Socket: sock, Hibernate: true}); err != nil {
+ t.Fatal(err)
+ }
+ rt, err := newTestRouter(reg, a)
+ if err != nil {
+ t.Fatalf("new router: %v", err)
+ }
+ front := httptest.NewServer(rt)
+ t.Cleanup(front.Close)
+
+ markedGet := func() int {
+ req, _ := http.NewRequest("GET", front.URL+"/", nil)
+ req.Host = host
+ req.Header.Set(SidecarHeader, "1")
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("marked request: %v", err)
+ }
+ defer resp.Body.Close()
+ io.Copy(io.Discard, resp.Body)
+ return resp.StatusCode
+ }
+
+ // A member request wakes the host.
+ if body, code := proxyGet(t, front.URL, host); code != 200 || body != "ok" {
+ t.Fatalf("member wake: got %d %q", code, body)
+ }
+ // Marked requests are served while awake…
+ if code := markedGet(); code != 200 {
+ t.Fatalf("marked request while awake: %d, want 200", code)
+ }
+ // …but keep stamping nothing: sidecar traffic riding through the whole
+ // idle window must not delay the sleep.
+ clk.advance(6 * time.Second)
+ if code := markedGet(); code != 200 {
+ t.Fatalf("marked request mid-window: %d, want 200", code)
+ }
+ clk.advance(5 * time.Second) // 11s since the member request
+ a.sweepOnce()
+ waitFor(t, "hibernation beneath sidecar traffic", func() bool { return a.entryCount() == 0 })
+
+ // Asleep: marked requests are refused and wake NOTHING — this is also the
+ // outsider-spoof case, and it's harmless.
+ if code := markedGet(); code != 503 {
+ t.Fatalf("marked request while asleep: %d, want 503", code)
+ }
+ if a.entryCount() != 0 {
+ t.Fatal("a marked request created an entry (a wake) for a sleeping host")
+ }
+ if _, _, starts := b.counts(); starts != 1 {
+ t.Fatalf("starts = %d after marked requests, want 1 (no sidecar-triggered wake)", starts)
+ }
+ // An unmarked request still wakes it, and the marker never reached the
+ // backend on any request.
+ if _, code := proxyGet(t, front.URL, host); code != 200 {
+ t.Fatalf("member re-wake: %d, want 200", code)
+ }
+ if _, _, starts := b.counts(); starts != 2 {
+ t.Fatalf("starts = %d after member re-wake, want 2", starts)
+ }
+ if leaked.Load() {
+ t.Fatal("the sidecar marker leaked through the proxy to the backend")
+ }
+}
+
+func TestConfigFromEnv(t *testing.T) {
+ t.Setenv("CARLOS_HIBERNATE_BUCKET", "")
+ if cfg, err := ConfigFromEnv(); err != nil || cfg != nil {
+ t.Fatalf("unset bucket: got %+v, %v; want nil, nil (feature off)", cfg, err)
+ }
+
+ t.Setenv("CARLOS_HIBERNATE_BUCKET", "b")
+ t.Setenv("CARLOS_HIBERNATE_REGION", "eu-west-1")
+ t.Setenv("CARLOS_HIBERNATE_PREFIX", "/instances/")
+ t.Setenv("CARLOS_HIBERNATE_IDLE", "90s")
+ cfg, err := ConfigFromEnv()
+ if err != nil {
+ t.Fatalf("config: %v", err)
+ }
+ if cfg.Bucket != "b" || cfg.Region != "eu-west-1" || cfg.Prefix != "instances" || cfg.Idle != 90*time.Second {
+ t.Fatalf("got %+v", cfg)
+ }
+ if got := cfg.hibernatePrefix("x.example.com"); got != "instances/x.example.com" {
+ t.Fatalf("prefix: %q", got)
+ }
+
+ t.Setenv("CARLOS_HIBERNATE_IDLE", "")
+ cfg, err = ConfigFromEnv()
+ if err != nil || cfg.Idle != 15*time.Minute {
+ t.Fatalf("default idle: %+v, %v", cfg, err)
+ }
+
+ t.Setenv("CARLOS_HIBERNATE_IDLE", "banana")
+ if _, err := ConfigFromEnv(); err == nil {
+ t.Fatal("bad idle duration should error, not default silently")
+ }
+}
+
+// --- ExecBackend -------------------------------------------------------------
+
+func testExecBackend(t *testing.T) *ExecBackend {
+ t.Helper()
+ return &ExecBackend{
+ Cfg: Config{Bucket: "test-bucket", Idle: time.Minute},
+ OptDir: t.TempDir(),
+ DataDir: t.TempDir(),
+ }
+}
+
+// stageLive materializes <OptDir>/live/<host> → <OptDir>/versions/<ver>/app,
+// the layout the adopt pass maintains in production.
+func stageLive(t *testing.T, b *ExecBackend, host, ver string) {
+ t.Helper()
+ vdir := filepath.Join(b.OptDir, "versions", ver)
+ if err := os.MkdirAll(vdir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ bin := filepath.Join(vdir, "app")
+ if err := os.WriteFile(bin, []byte("#!/bin/sh\n"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.MkdirAll(filepath.Join(b.OptDir, "live"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ os.Remove(b.liveLink(host))
+ if err := os.Symlink(bin, b.liveLink(host)); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// Already-materialized: the common case for every existing instance — the
+// Materialize hook is never invoked.
+func TestExecBackendLiveLinkAlreadyMaterialized(t *testing.T) {
+ b := testExecBackend(t)
+ stageLive(t, b, "existing.example.com", "v1")
+ called := false
+ b.Materialize = func(context.Context, string) error { called = true; return nil }
+ if err := b.ensureLiveLink(context.Background(), "existing.example.com"); err != nil {
+ t.Fatalf("ensureLiveLink: %v", err)
+ }
+ if called {
+ t.Error("Materialize was called for an already-materialized host")
+ }
+}
+
+// Missing symlink + Materialize configured: routed through the hook, and the
+// hook's result is honored.
+func TestExecBackendLiveLinkUsesMaterializeWhenConfigured(t *testing.T) {
+ b := testExecBackend(t)
+ var gotHost string
+ b.Materialize = func(_ context.Context, host string) error {
+ gotHost = host
+ stageLive(t, b, host, "v1")
+ return nil
+ }
+ if err := b.ensureLiveLink(context.Background(), "seeded.example.com"); err != nil {
+ t.Fatalf("ensureLiveLink: %v", err)
+ }
+ if gotHost != "seeded.example.com" {
+ t.Fatalf("Materialize called with %q, want seeded.example.com", gotHost)
+ }
+ if _, err := os.Stat(b.liveLink("seeded.example.com")); err != nil {
+ t.Fatalf("live symlink was not created via Materialize: %v", err)
+ }
+}
+
+// A Materialize failure must propagate — the exact live symptom found
+// 2026-07-13 in Eleven (a 503 "instance failed to wake"), never silently
+// swallowed into "looks fine, wakes anyway."
+func TestExecBackendLiveLinkPropagatesMaterializeError(t *testing.T) {
+ b := testExecBackend(t)
+ wantErr := fmt.Errorf("controld materialize seeded.example.com: HTTP 502")
+ b.Materialize = func(context.Context, string) error { return wantErr }
+ err := b.ensureLiveLink(context.Background(), "seeded.example.com")
+ if err == nil {
+ t.Fatal("expected an error, got nil")
+ }
+ if !errors.Is(err, wantErr) {
+ t.Fatalf("got %v, want it to wrap %v", err, wantErr)
+ }
+ if _, statErr := os.Stat(b.liveLink("seeded.example.com")); statErr == nil {
+ t.Fatal("live symlink should not exist after a failed Materialize")
+ }
+}
+
+// No hook and no symlink: an error, loudly. (Eleven self-staged its own
+// binary here — instance and control binary were one program; a carlos app is
+// a separate binary from the release store, so there is nothing to invent.)
+func TestExecBackendLiveLinkMissingWithoutMaterializeFails(t *testing.T) {
+ b := testExecBackend(t)
+ if err := b.ensureLiveLink(context.Background(), "ghost.example.com"); err == nil {
+ t.Fatal("expected an error for a missing live symlink with no Materialize hook")
+ }
+}
+
+func TestExecBackendLiveVersion(t *testing.T) {
+ b := testExecBackend(t)
+ stageLive(t, b, "v.example.com", "abc1234")
+ ver, err := b.LiveVersion("v.example.com")
+ if err != nil || ver != "abc1234" {
+ t.Fatalf("LiveVersion = %q, %v; want abc1234", ver, err)
+ }
+ // No symlink at all: "" with no error — version management not set up yet.
+ ver, err = b.LiveVersion("none.example.com")
+ if err != nil || ver != "" {
+ t.Fatalf("LiveVersion(no link) = %q, %v; want \"\", nil", ver, err)
+ }
+}
+
+// childEnv strips every CARLOS_* var from the inherited environment (the
+// edge's own config must never leak into a child) and then layers the env
+// file on top — including that file's own CARLOS_* entries.
+func TestChildEnvStripsCarlosVars(t *testing.T) {
+ t.Setenv("CARLOS_HIBERNATE_BUCKET", "leak-me-not")
+ t.Setenv("CARLOS_TEST_HARMLESS", "also-me")
+ t.Setenv("UNRELATED_VAR", "keep")
+ env := childEnv(map[string]string{"CARLOS_ADMIN_TOKEN": "tok", "FOO": "bar"})
+ got := map[string]string{}
+ for _, kv := range env {
+ if k, v, ok := splitEnv(kv); ok {
+ got[k] = v
+ }
+ }
+ if _, leaked := got["CARLOS_HIBERNATE_BUCKET"]; leaked {
+ t.Error("inherited CARLOS_HIBERNATE_BUCKET leaked into the child env")
+ }
+ if _, leaked := got["CARLOS_TEST_HARMLESS"]; leaked {
+ t.Error("inherited CARLOS_TEST_HARMLESS leaked into the child env")
+ }
+ if got["UNRELATED_VAR"] != "keep" {
+ t.Error("non-CARLOS inherited var was stripped")
+ }
+ if got["CARLOS_ADMIN_TOKEN"] != "tok" || got["FOO"] != "bar" {
+ t.Errorf("env file vars not layered: %v", got)
+ }
+}
+
+func splitEnv(kv string) (k, v string, ok bool) {
+ for i := 0; i < len(kv); i++ {
+ if kv[i] == '=' {
+ return kv[:i], kv[i+1:], true
+ }
+ }
+ return "", "", false
+}
+
+// The sleep-time next-due probe: GET /api/next-due over the instance's unix
+// socket, authorized with the CARLOS_ADMIN_TOKEN from its env file.
+func TestExecBackendNextDue(t *testing.T) {
+ b := testExecBackend(t)
+ sock := tmpSock(t)
+ ln, err := net.Listen("unix", sock)
+ if err != nil {
+ t.Fatalf("listen unix: %v", err)
+ }
+ srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/api/next-due" {
+ http.NotFound(w, r)
+ return
+ }
+ if r.Header.Get("Authorization") != "Bearer tok" {
+ http.Error(w, "forbidden", http.StatusForbidden)
+ return
+ }
+ io.WriteString(w, `{"due": 42}`)
+ })}
+ go srv.Serve(ln)
+ t.Cleanup(func() { srv.Close() })
+
+ host := "due.example.com"
+ rt := registry.Route{Host: host, Kind: "instance", Socket: sock, Hibernate: true}
+
+ // No env file at all: the probe fails (the state machine records 0).
+ if _, err := b.NextDue(context.Background(), rt); err == nil {
+ t.Fatal("NextDue with no env file should error")
+ }
+ // Env file without a token: same.
+ envPath := filepath.Join(b.DataDir, host+".env")
+ if err := os.WriteFile(envPath, []byte("# no token here\nFOO=bar\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := b.NextDue(context.Background(), rt); err == nil {
+ t.Fatal("NextDue without CARLOS_ADMIN_TOKEN should error")
+ }
+ // With the token: the instance's answer comes back.
+ if err := os.WriteFile(envPath, []byte("CARLOS_ADMIN_TOKEN=tok\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ due, err := b.NextDue(context.Background(), rt)
+ if err != nil || due != 42 {
+ t.Fatalf("NextDue = %d, %v; want 42", due, err)
+ }
+}
+
+// HasSidecar is derived from the filesystem — the /data/<host>-sidecar.env
+// convention — no registry column to drift from what's actually staged.
+func TestExecBackendHasSidecar(t *testing.T) {
+ b := testExecBackend(t)
+ if b.HasSidecar("plain.example.com") {
+ t.Fatal("HasSidecar true with no env file staged")
+ }
+ if err := os.WriteFile(filepath.Join(b.DataDir, "bots.example.com-sidecar.env"), []byte("CARLOS_BASE=x\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if !b.HasSidecar("bots.example.com") {
+ t.Fatal("HasSidecar false with the sidecar env file staged")
+ }
+}
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
new file mode 100644
index 000000000..37df59d7e
--- /dev/null
+++ b/internal/activator/backend_exec.go
@@ -0,0 +1,435 @@
+// Ported from lchat activator.go (the s3Backend half) and the exec/env/health
+// helpers it leaned on from lchat control.go and versions.go.
+package activator
+
+import (
+ "bufio"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "net"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// spawnHealthTimeout bounds how long a spawned instance may take to bind its
+// socket and answer. A var so tests can shrink the failure path.
+var spawnHealthTimeout = 30 * time.Second
+
+// ExecBackend is the production Backend: leases + restore via the lease
+// package's litestream primitives, processes via exec of the same versioned
+// binary a systemd unit would run — `<OptDir>/live/<host>` with the
+// `<DataDir>/<host>.env` EnvironmentFile.
+type ExecBackend struct {
+ Cfg Config
+ // OptDir is the on-box install root: the version store and the
+ // per-instance "live" symlinks live under it (/opt/carlos in production;
+ // a struct field, never hardcoded, so tests point it at a temp dir).
+ OptDir string
+ // DataDir holds each instance's DB and env files (<host>.db, <host>.env,
+ // <host>-sidecar.env) — /data in production.
+ DataDir string
+ // Materialize, when non-nil, stages a binary and points the host's live
+ // symlink at it via an external mechanism instead of the backend doing
+ // filesystem work itself — Eleven needed this because its router ran
+ // sandboxed (ProtectSystem=strict) and couldn't write to /opt itself.
+ // With it nil, a missing live symlink is an error: Eleven fell back to
+ // staging its OWN binary (instance and control binary were one program),
+ // but a carlos app is a separate binary from the release store — there is
+ // nothing sensible to invent here, and the adopt pass owns the symlinks.
+ Materialize func(ctx context.Context, host string) error
+}
+
+var _ Backend = (*ExecBackend)(nil)
+
+func (b *ExecBackend) liveDir() string { return filepath.Join(b.OptDir, "live") }
+func (b *ExecBackend) liveLink(host string) string { return filepath.Join(b.liveDir(), host) }
+func (b *ExecBackend) defaultDBPath(host string) string {
+ return filepath.Join(b.DataDir, host+".db")
+}
+
+// LiveVersion reports the version host's live symlink currently points at, or
+// "" (no error) if the host hasn't been put under version management yet.
+// (target = <OptDir>/versions/<ver>/<app> → the parent dir's name is <ver>.)
+// It doubles as the Service's convergence-sweep seam — New picks it up via an
+// optional interface.
+func (b *ExecBackend) LiveVersion(host string) (string, error) {
+ target, err := os.Readlink(b.liveLink(host))
+ if err != nil {
+ if os.IsNotExist(err) {
+ return "", nil
+ }
+ return "", err
+ }
+ return filepath.Base(filepath.Dir(target)), nil
+}
+
+// Acquire claims the per-host single-writer lease. A contested lease is
+// re-surfaced as errLeaseHeld (never a litestream type — the lease package's
+// *HeldError is this package's only view of it) so the edge can 503 with a
+// Retry-After that tracks the holder's expiry.
+func (b *ExecBackend) Acquire(ctx context.Context, host string) (Lease, error) {
+ held, err := lease.Acquire(ctx, b.Cfg.Bucket, b.Cfg.hibernatePrefix(host), b.Cfg.Region)
+ if err != nil {
+ var he *lease.HeldError
+ if errors.As(err, &he) {
+ retry := time.Until(he.ExpiresAt)
+ if retry < time.Second {
+ retry = time.Second
+ }
+ return nil, &errLeaseHeld{holder: he.Owner, retryAfter: retry}
+ }
+ return nil, err
+ }
+ return &execLease{held: held, defaultDB: b.defaultDBPath(host)}, nil
+}
+
+// execLease adapts *lease.Held to the Lease seam.
+type execLease struct {
+ held *lease.Held
+ defaultDB string // used when the route carries no DBPath
+}
+
+func (l *execLease) Replicate(ctx context.Context, dbPath string, syncInterval time.Duration) (Rep, error) {
+ if dbPath == "" {
+ dbPath = l.defaultDB
+ }
+ rep, err := l.held.Replicate(ctx, dbPath, syncInterval)
+ if err != nil {
+ return nil, err
+ }
+ return rep, nil
+}
+
+func (l *execLease) Release(ctx context.Context) error { return l.held.Release(ctx) }
+
+func (b *ExecBackend) Restore(ctx context.Context, host, dbPath string) error {
+ if dbPath == "" {
+ dbPath = b.defaultDBPath(host)
+ }
+ if _, err := os.Stat(dbPath); err == nil {
+ return nil // warm cache — see doWake
+ } else if !os.IsNotExist(err) {
+ return err
+ }
+ // The DB is gone, so any leftover litestream sidecar state (meta dirs,
+ // WAL/SHM) describes a database that no longer exists — litestream's own
+ // recovery guidance is to delete it. Clear it so the restored copy starts
+ // a clean lineage. Two meta dirs: litestream's default suffix ("-litestream",
+ // its MetaDirSuffix — spelled out because this package never imports
+ // litestream) as any box-wide backup replicator would use, and the lease
+ // package's activator-scoped ".activator-<db>-litestream".
+ for _, p := range []string{dbPath + "-wal", dbPath + "-shm"} {
+ _ = os.Remove(p)
+ }
+ _ = os.RemoveAll(dbPath + "-litestream")
+ _ = os.RemoveAll(filepath.Join(filepath.Dir(dbPath), ".activator-"+filepath.Base(dbPath)+"-litestream"))
+
+ restored, err := lease.Restore(ctx, dbPath, b.Cfg.Bucket, b.Cfg.hibernatePrefix(host), b.Cfg.Region)
+ if err != nil {
+ return err
+ }
+ if restored {
+ log.Printf("activator: restored %s from s3://%s/%s", host, b.Cfg.Bucket, b.Cfg.hibernatePrefix(host))
+ } else {
+ // Never replicated: a brand-new instance boots fresh and creates its
+ // own DB — the register-only create path (provisioning as a metadata
+ // write) landing on its first wake.
+ log.Printf("activator: no replica for %s in s3://%s/%s — booting fresh", host, b.Cfg.Bucket, b.Cfg.hibernatePrefix(host))
+ }
+ return nil
+}
+
+// NextDue asks the (still-running) instance for its earliest pending scheduled
+// delivery over its socket. The endpoint is admin-gated like every instance
+// data endpoint, so present the instance's own admin token — read from the
+// same env file the spawn path hands the child (the activator holds it either
+// way).
+func (b *ExecBackend) NextDue(ctx context.Context, rt registry.Route) (int64, error) {
+ fileEnv, err := readEnvFile(filepath.Join(b.DataDir, rt.Host+".env"))
+ if err != nil {
+ return 0, fmt.Errorf("read env file: %w", err)
+ }
+ token := fileEnv["CARLOS_ADMIN_TOKEN"]
+ if token == "" {
+ return 0, errors.New("no CARLOS_ADMIN_TOKEN in env file")
+ }
+ req, err := http.NewRequestWithContext(ctx, "GET", "http://carlos/api/next-due", nil)
+ if err != nil {
+ return 0, err
+ }
+ req.Header.Set("Authorization", "Bearer "+token)
+ resp, err := socketClient(rt.Socket).Do(req)
+ if err != nil {
+ return 0, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return 0, fmt.Errorf("GET /api/next-due: HTTP %d", resp.StatusCode)
+ }
+ var body struct {
+ Due int64 `json:"due"`
+ }
+ if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<12)).Decode(&body); err != nil {
+ return 0, fmt.Errorf("decode /api/next-due: %w", err)
+ }
+ return body.Due, nil
+}
+
+// ensureLiveLink verifies host's live symlink (<OptDir>/live/<host> → a staged
+// version) exists before the exec, routing a missing one through the
+// Materialize hook when configured. A materialize failure (e.g. the staging
+// mechanism unreachable) must propagate — the exact live symptom found
+// 2026-07-13 in Eleven (a 503 "instance failed to wake"), never silently
+// swallowed into "looks fine, wakes anyway."
+func (b *ExecBackend) ensureLiveLink(ctx context.Context, host string) error {
+ bin := b.liveLink(host)
+ if _, err := os.Stat(bin); err == nil {
+ return nil
+ }
+ if b.Materialize != nil {
+ if err := b.Materialize(ctx, host); err != nil {
+ return fmt.Errorf("materialize live symlink for %s: %w", host, err)
+ }
+ return nil
+ }
+ return fmt.Errorf("no live binary for %s (%s missing and no Materialize hook)", host, bin)
+}
+
+func (b *ExecBackend) Start(ctx context.Context, rt registry.Route) (Proc, string, error) {
+ host := rt.Host
+ dbPath := rt.DBPath
+ if dbPath == "" {
+ dbPath = b.defaultDBPath(host)
+ }
+
+ // Exec exactly what a systemd unit would: the per-instance live symlink
+ // (<OptDir>/live/<host> → a staged version), same --socket/--db flags,
+ // same EnvironmentFile.
+ bin := b.liveLink(host)
+ if err := b.ensureLiveLink(ctx, host); err != nil {
+ return nil, "", err
+ }
+
+ // The env file is optional, like the unit's "-" prefix.
+ fileEnv, err := readEnvFile(filepath.Join(b.DataDir, host+".env"))
+ if err != nil && !os.IsNotExist(err) {
+ return nil, "", fmt.Errorf("read env file: %w", err)
+ }
+
+ cmd := exec.Command(bin, "--socket", rt.Socket, "--db", dbPath)
+ cmd.Env = childEnv(fileEnv)
+ // The child logs through the agent's own stderr/stdout (journald under
+ // systemd); there's no per-instance unit to own a log stream anymore.
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ // Record the build we're about to exec — read the symlink immediately before
+ // Start() so it names the version the kernel will resolve. Reading it here,
+ // rather than back in wake() after the health wait, closes a real window: a
+ // concurrent ship's repoint during the up-to-30s wait would otherwise make
+ // us record the NEW target while this child runs the OLD binary, and the
+ // sweep would then think it's already converged and never cycle it. The
+ // residual sub-exec race (a repoint between this read and Start) is in the
+ // safe direction — we'd record the older version, so the sweep cycles once
+ // too often rather than never. Empty on a read error: the sweep won't cycle.
+ spawnVer, _ := b.LiveVersion(host)
+ if err := cmd.Start(); err != nil {
+ return nil, "", err
+ }
+
+ p := &childProc{cmd: cmd, done: make(chan struct{})}
+ go func() {
+ p.err = cmd.Wait()
+ close(p.done)
+ }()
+
+ if err := waitHealthyCtx(ctx, rt.Socket, spawnHealthTimeout); err != nil {
+ stopCtx, cancel := context.WithTimeout(context.Background(), actStopTimeout)
+ defer cancel()
+ _ = p.Stop(stopCtx)
+ return nil, "", err
+ }
+ return p, spawnVer, nil
+}
+
+// childEnv approximates a systemd unit's environment from inside the agent:
+// inherit the process env but strip every CARLOS_* var first, so the agent's
+// own config (hibernation, registry, tokens) can never leak into a child;
+// then layer the child's EnvironmentFile on top. A nil fileEnv (no env file,
+// like the unit's "-" prefix) yields just the cleaned inherit.
+func childEnv(fileEnv map[string]string) []string {
+ env := make([]string, 0, 32)
+ for _, kv := range os.Environ() {
+ if strings.HasPrefix(kv, "CARLOS_") {
+ continue
+ }
+ env = append(env, kv)
+ }
+ for k, v := range fileEnv {
+ env = append(env, k+"="+v)
+ }
+ return env
+}
+
+// sidecarEnvPath: the host's sidecar EnvironmentFile,
+// <DataDir>/<host>-sidecar.env (Eleven's <host>-bot.env, renamed).
+func (b *ExecBackend) sidecarEnvPath(host string) string {
+ return filepath.Join(b.DataDir, host+"-sidecar.env")
+}
+
+// HasSidecar: a host has a sidecar iff its sidecar EnvironmentFile exists.
+// Derived from the filesystem on purpose, the way systemd units would do it:
+// no registry column to drift from what's actually staged.
+func (b *ExecBackend) HasSidecar(host string) bool {
+ _, err := os.Stat(b.sidecarEnvPath(host))
+ return err == nil
+}
+
+// StartSidecar execs the host's live symlink with `sidecar run` under the
+// sidecar's EnvironmentFile, CARLOS_*-clean like the instance spawn. No
+// health wait — the sidecar is a pure outbound client with no socket to
+// probe; its own connect/backoff loop absorbs a slow instance. The live
+// symlink exists by now: the instance spawn checked it. The ctx is unused (an
+// exec doesn't block); it's here for the backend seam's symmetry.
+func (b *ExecBackend) StartSidecar(_ context.Context, rt registry.Route) (Proc, error) {
+ fileEnv, err := readEnvFile(b.sidecarEnvPath(rt.Host))
+ if err != nil {
+ return nil, fmt.Errorf("read sidecar env: %w", err)
+ }
+ cmd := exec.Command(b.liveLink(rt.Host), "sidecar", "run")
+ cmd.Env = childEnv(fileEnv)
+ // Like the instance child: logs through the agent's own streams (journald
+ // under systemd) — no per-host unit owns a log stream anymore.
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ if err := cmd.Start(); err != nil {
+ return nil, err
+ }
+ p := &childProc{cmd: cmd, done: make(chan struct{})}
+ go func() {
+ p.err = cmd.Wait()
+ close(p.done)
+ }()
+ return p, nil
+}
+
+// socketClient returns an HTTP client that dials an instance's unix socket
+// (request URLs use the placeholder host http://carlos).
+func socketClient(socket string) *http.Client {
+ return &http.Client{
+ Timeout: 3 * time.Second,
+ Transport: &http.Transport{
+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+ return (&net.Dialer{}).DialContext(ctx, "unix", socket)
+ },
+ },
+ }
+}
+
+// waitHealthy blocks until the instance answers an HTTP request on its socket,
+// or the timeout elapses. It retries, so it's robust to the restart window
+// (socket briefly absent, or a stale socket left by an abruptly-killed old
+// process) — and a broken binary that never becomes healthy fails the wake.
+func waitHealthy(socket string, timeout time.Duration) error {
+ deadline := time.Now().Add(timeout)
+ client := socketClient(socket)
+ var last error
+ for time.Now().Before(deadline) {
+ resp, err := client.Get("http://carlos/")
+ if err == nil {
+ resp.Body.Close()
+ if resp.StatusCode < 500 {
+ return nil
+ }
+ last = fmt.Errorf("unhealthy: HTTP %d", resp.StatusCode)
+ } else {
+ last = err
+ }
+ time.Sleep(150 * time.Millisecond)
+ }
+ return fmt.Errorf("instance not healthy within %s: %v", timeout, last)
+}
+
+// waitHealthyCtx is waitHealthy bounded by ctx as well as its own timeout, so
+// an aborted wake doesn't keep probing a socket nobody wants anymore.
+func waitHealthyCtx(ctx context.Context, socket string, timeout time.Duration) error {
+ done := make(chan error, 1)
+ go func() { done <- waitHealthy(socket, timeout) }()
+ select {
+ case err := <-done:
+ return err
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+// readEnvFile parses a KEY=value EnvironmentFile (blank lines and #comments
+// ignored).
+func readEnvFile(path string) (map[string]string, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ m := map[string]string{}
+ sc := bufio.NewScanner(f)
+ for sc.Scan() {
+ line := strings.TrimSpace(sc.Text())
+ if line == "" || strings.HasPrefix(line, "#") {
+ continue
+ }
+ if k, v, ok := strings.Cut(line, "="); ok {
+ m[strings.TrimSpace(k)] = strings.TrimSpace(v)
+ }
+ }
+ return m, sc.Err()
+}
+
+// childProc is a spawned child: SIGTERM starts the instance's own graceful
+// drain; SIGKILL only if it overstays.
+type childProc struct {
+ cmd *exec.Cmd
+ done chan struct{}
+ err error
+}
+
+func (p *childProc) Exited() bool {
+ select {
+ case <-p.done:
+ return true
+ default:
+ return false
+ }
+}
+
+func (p *childProc) Stop(ctx context.Context) error {
+ if p.Exited() {
+ return nil
+ }
+ if err := p.cmd.Process.Signal(syscall.SIGTERM); err != nil {
+ if errors.Is(err, os.ErrProcessDone) { // lost the race with its own exit
+ <-p.done
+ return nil
+ }
+ return err
+ }
+ select {
+ case <-p.done:
+ return nil
+ case <-ctx.Done():
+ _ = p.cmd.Process.Kill()
+ <-p.done
+ return fmt.Errorf("instance did not exit on SIGTERM; killed")
+ }
+}
diff --git a/internal/ops/litestream.go b/internal/ops/litestream.go
new file mode 100644
index 000000000..590ef6825
--- /dev/null
+++ b/internal/ops/litestream.go
@@ -0,0 +1,210 @@
+// litestream — off-box replication of the fleet's SQLite files, and the daily
+// proof that the replicas restore. Two verbs, both run ON the box (titogo's
+// operator-side SSH orchestration deliberately did not come along — units and
+// credentials are provisioned by bootstrap; this is only the box-side half):
+//
+// carlos ops litestream (re)generate /etc/litestream.yml from the
+// registry + data dir; prints "changed" or
+// "unchanged" so the calling timer/unit knows
+// whether a replicator restart is due
+// carlos ops restore-verify restore every configured database from S3 and
+// integrity-check it
+//
+// The config is always generated, never hand-maintained: `carlos add`/`remove`
+// change the database set, and re-running the verb is the one way the
+// replication set follows. Credentials live only in the unit's EnvironmentFile
+// (0600) — this code never sees or prints them.
+//
+// Ported from titogo internal/ops/litestream.go (config generator, round-trip
+// parser, restore-verify), plus Eleven deploy/fleet-backup.sh's hibernation
+// exclusion.
+package ops
+
+import (
+ "bytes"
+ "database/sql"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+
+ _ "modernc.org/sqlite"
+)
+
+// litestreamVersion is the pinned release provisioned on the box when the
+// binary is missing. Bump deliberately; the restore path is the thing a
+// version change can silently break, and restore-verify is what catches it.
+const litestreamVersion = "0.5.15"
+
+// retryDelay is the pause before restore-verify's one retry; a var so tests
+// can shrink it.
+var retryDelay = 10 * time.Second
+
+// LitestreamConfig renders the generated /etc/litestream.yml. Deliberately
+// minimal — path + replica URL + region only — so it can't rot against
+// litestream's config surface; tuning belongs in flags on the unit, not here.
+// Paths are sorted so re-runs with an unchanged fleet produce a byte-identical
+// file (the `litestream` verb's changed/unchanged report depends on it).
+func LitestreamConfig(dbPaths []string, bucket, prefix, region string) string {
+ sorted := append([]string(nil), dbPaths...)
+ sort.Strings(sorted)
+ var b strings.Builder
+ b.WriteString("# GENERATED by `carlos ops litestream` — do not hand-edit; re-run the verb\n")
+ b.WriteString("# after `carlos add`/`remove` so the replication set follows the fleet.\n")
+ b.WriteString("dbs:\n")
+ for _, p := range sorted {
+ fmt.Fprintf(&b, " - path: %s\n", p)
+ b.WriteString(" replicas:\n")
+ fmt.Fprintf(&b, " - url: s3://%s/%s/%s\n", bucket, prefix, filepath.Base(p))
+ if region != "" {
+ fmt.Fprintf(&b, " region: %s\n", region)
+ }
+ }
+ return b.String()
+}
+
+// ParseLitestreamDBs recovers the database paths from a generated config.
+// It reads only the exact shape LitestreamConfig writes (the round-trip is
+// pinned by TestConfigRoundTrip) — it is not a YAML parser.
+func ParseLitestreamDBs(cfg string) []string {
+ var dbs []string
+ for _, line := range strings.Split(cfg, "\n") {
+ if p, ok := strings.CutPrefix(line, " - path: "); ok {
+ dbs = append(dbs, strings.TrimSpace(p))
+ }
+ }
+ return dbs
+}
+
+// FleetDBs lists the non-empty *.db files under dataDir that this replicator
+// owns. Hibernating routes' databases (matched by DBPath) are excluded:
+// the activator's embedded Litestream owns their replication (to the
+// instances bucket, under the lease) — a second replicator on the same WAL
+// would fight it for checkpointing and corrupt the replica. (Eleven's
+// fleet-backup.sh lesson, carried verbatim.)
+func FleetDBs(reg *registry.Registry, dataDir string) ([]string, error) {
+ routes, err := reg.List()
+ if err != nil {
+ return nil, err
+ }
+ hibernating := map[string]bool{}
+ for _, r := range routes {
+ if r.Hibernate && r.DBPath != "" {
+ hibernating[filepath.Clean(r.DBPath)] = true
+ }
+ }
+
+ matches, err := filepath.Glob(filepath.Join(dataDir, "*.db"))
+ if err != nil {
+ return nil, err
+ }
+ var dbs []string
+ for _, p := range matches {
+ fi, err := os.Stat(p)
+ if err != nil || !fi.Mode().IsRegular() || fi.Size() == 0 {
+ continue // gone, odd, or an empty placeholder — nothing to replicate
+ }
+ if hibernating[filepath.Clean(p)] {
+ continue
+ }
+ dbs = append(dbs, p)
+ }
+ return dbs, nil
+}
+
+// RestoreVerify restores every database in the generated config from S3 into
+// tmpDir and proves each opens clean. Returns an error on any failure so the
+// caller (a oneshot unit) can exit non-zero where the fleet sweeps see it.
+func RestoreVerify(litestreamBin, cfgPath, tmpDir string) error {
+ raw, err := os.ReadFile(cfgPath)
+ if err != nil {
+ return fmt.Errorf("restore-verify: %v", err)
+ }
+ dbs := ParseLitestreamDBs(string(raw))
+ if len(dbs) == 0 {
+ return fmt.Errorf("restore-verify: no databases in %s — regenerate it: carlos ops litestream", cfgPath)
+ }
+
+ failures := 0
+ for _, db := range dbs {
+ if err := restoreAndCheck(litestreamBin, cfgPath, db, tmpDir); err != nil {
+ // One retry after a pause: a database created moments ago may not
+ // have finished its first replication pass yet.
+ time.Sleep(retryDelay)
+ err = restoreAndCheck(litestreamBin, cfgPath, db, tmpDir)
+ if err != nil {
+ failures++
+ fmt.Printf("FAIL %s: %v\n", db, err)
+ continue
+ }
+ }
+ fmt.Printf("ok %s\n", db)
+ }
+ if failures > 0 {
+ return fmt.Errorf("restore-verify: %d of %d replicas failed to restore clean", failures, len(dbs))
+ }
+ fmt.Printf("restore-verify: all %d replicas restore clean\n", len(dbs))
+ return nil
+}
+
+// restoreAndCheck pulls one database back from its replica and runs the same
+// checks a disaster recovery would lean on: the file opens, integrity_check
+// says ok, and the schema is non-empty.
+func restoreAndCheck(lsBin, cfgPath, dbPath, tmpDir string) error {
+ out := filepath.Join(tmpDir, filepath.Base(dbPath))
+ _ = os.Remove(out) // a retry must not trip over the failed attempt's file
+ cmd := exec.Command(lsBin, "restore", "-config", cfgPath, "-o", out, dbPath)
+ if b, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("litestream restore: %v: %s", err, strings.TrimSpace(string(b)))
+ }
+ return CheckRestoredDB(out)
+}
+
+// CheckRestoredDB opens a restored file read-only and proves it is a sane
+// SQLite database: integrity_check == ok and at least one table present.
+func CheckRestoredDB(path string) error {
+ db, err := sql.Open("sqlite", "file:"+path+"?mode=ro")
+ if err != nil {
+ return err
+ }
+ defer db.Close()
+ db.SetMaxOpenConns(1)
+ // busy_timeout first, always (Global Constraints) — and on a garbage file
+ // this first statement is what fails loudly instead of a later query.
+ if _, err := db.Exec("PRAGMA busy_timeout=5000"); err != nil {
+ return fmt.Errorf("open: %v", err)
+ }
+ var res string
+ if err := db.QueryRow("PRAGMA integrity_check").Scan(&res); err != nil {
+ return fmt.Errorf("integrity_check: %v", err)
+ }
+ if res != "ok" {
+ return fmt.Errorf("integrity_check: %s", res)
+ }
+ var tables int
+ if err := db.QueryRow("SELECT count(*) FROM sqlite_master WHERE type='table'").Scan(&tables); err != nil {
+ return fmt.Errorf("schema count: %v", err)
+ }
+ if tables == 0 {
+ return fmt.Errorf("restored database has no tables")
+ }
+ return nil
+}
+
+// writeIfChanged writes content to path only when it differs, and reports
+// whether it wrote. The byte-identical no-op is what lets a timer re-run the
+// generator freely without a restart-worthy mtime/content churn.
+func writeIfChanged(path, content string) (bool, error) {
+ if old, err := os.ReadFile(path); err == nil && bytes.Equal(old, []byte(content)) {
+ return false, nil
+ }
+ if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+ return false, err
+ }
+ return true, nil
+}
diff --git a/internal/ops/ops.go b/internal/ops/ops.go
new file mode 100644
index 000000000..bf7d6668e
--- /dev/null
+++ b/internal/ops/ops.go
@@ -0,0 +1,139 @@
+// Package ops is the box-side operations surface — verbs a systemd timer (or
+// an operator's shell on the box) runs to keep replication honest. Inspired by
+// titogo internal/ops: every verb encodes knowledge that otherwise lives only
+// in prose, and prose drifts silently where a command that runs drifts loudly.
+package ops
+
+import (
+ "flag"
+ "fmt"
+ "log"
+ "os"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+const (
+ defaultConfigPath = "/etc/litestream.yml"
+ defaultDataDir = "/data"
+)
+
+// Run dispatches `carlos ops <verb>`.
+func Run(args []string) {
+ verb := ""
+ if len(args) > 0 {
+ verb = args[0]
+ args = args[1:]
+ }
+ switch verb {
+ case "litestream":
+ runLitestream(args)
+ case "restore-verify":
+ runRestoreVerify(args)
+ case "", "-h", "--help", "help":
+ usage()
+ if verb == "" {
+ os.Exit(2)
+ }
+ default:
+ fmt.Fprintf(os.Stderr, "ops: unknown verb %q\n\n", verb)
+ usage()
+ os.Exit(2)
+ }
+}
+
+func usage() {
+ fmt.Fprintf(os.Stderr, `carlos ops — box-side operations verbs
+
+ carlos ops litestream (re)generate the fleet litestream config from the
+ registry + data dir; prints "changed"/"unchanged"
+ so the caller knows whether to restart replication
+ carlos ops restore-verify restore every configured replica from S3 and
+ integrity-check each (run by the daily timer)
+
+No verb for your task? Extend ops in a small PR — never a hand-run sequence
+reconstructed from docs.
+`)
+}
+
+// dataDir resolves the data directory: flag wins, then CARLOS_DATA_DIR, then
+// the fleet default.
+func dataDir(flagVal string) string {
+ if flagVal != "" {
+ return flagVal
+ }
+ if env := os.Getenv("CARLOS_DATA_DIR"); env != "" {
+ return env
+ }
+ return defaultDataDir
+}
+
+// runLitestream — regenerate the fleet replication config. Idempotent: run it
+// after every `carlos add`/`remove` (or on a timer; a no-op re-run is cheap
+// and prints "unchanged"). Databases of hibernating routes are excluded — see
+// FleetDBs for Eleven's two-replicators lesson.
+func runLitestream(args []string) {
+ fs := flag.NewFlagSet("ops litestream", flag.ExitOnError)
+ regPath := fs.String("registry", "", "registry database (default <data>/registry.db)")
+ data := fs.String("data", "", "data directory holding the fleet's *.db files (default $CARLOS_DATA_DIR or /data)")
+ bucket := fs.String("bucket", "", "S3 bucket receiving the replicas (required)")
+ prefix := fs.String("prefix", "", "key prefix inside the bucket for this fleet (required)")
+ region := fs.String("region", "", "S3 region for the replica stanza")
+ out := fs.String("out", defaultConfigPath, "config file to write")
+ _ = fs.Parse(args)
+
+ if *bucket == "" || *prefix == "" {
+ log.Fatal("litestream: -bucket and -prefix are required")
+ }
+ dir := dataDir(*data)
+ rp := *regPath
+ if rp == "" {
+ rp = dir + "/registry.db"
+ }
+ reg, err := registry.Open(rp)
+ if err != nil {
+ log.Fatalf("litestream: open registry %s: %v", rp, err)
+ }
+ dbs, err := FleetDBs(reg, dir)
+ if err != nil {
+ log.Fatalf("litestream: %v", err)
+ }
+ if len(dbs) == 0 {
+ log.Fatalf("litestream: no non-empty *.db files under %s — wrong box?", dir)
+ }
+
+ cfg := LitestreamConfig(dbs, *bucket, *prefix, *region)
+ wrote, err := writeIfChanged(*out, cfg)
+ if err != nil {
+ log.Fatalf("litestream: write %s: %v", *out, err)
+ }
+ fmt.Fprintf(os.Stderr, "litestream: %d databases -> s3://%s/%s/ (%s)\n", len(dbs), *bucket, *prefix, *out)
+ // The one contract line on stdout: the calling unit restarts the
+ // replicator on "changed" and leaves it alone on "unchanged".
+ if wrote {
+ fmt.Println("changed")
+ } else {
+ fmt.Println("unchanged")
+ }
+}
+
+// runRestoreVerify — restore every database in the generated config from S3
+// into a temp dir and prove each opens clean. Exits non-zero on any failure so
+// the oneshot unit lands in 'failed' where the fleet sweeps see it.
+func runRestoreVerify(args []string) {
+ fs := flag.NewFlagSet("ops restore-verify", flag.ExitOnError)
+ cfgPath := fs.String("config", defaultConfigPath, "generated litestream config to read the database set from")
+ lsBin := fs.String("litestream-bin", "litestream", "litestream binary (pinned release "+litestreamVersion+")")
+ _ = fs.Parse(args)
+
+ tmp, err := os.MkdirTemp("", "carlos-restore")
+ if err != nil {
+ log.Fatalf("restore-verify: %v", err)
+ }
+ defer os.RemoveAll(tmp)
+
+ if err := RestoreVerify(*lsBin, *cfgPath, tmp); err != nil {
+ os.RemoveAll(tmp) // log.Fatal skips the defer
+ log.Fatal(err)
+ }
+}
diff --git a/internal/ops/ops_test.go b/internal/ops/ops_test.go
new file mode 100644
index 000000000..bda9689d1
--- /dev/null
+++ b/internal/ops/ops_test.go
@@ -0,0 +1,144 @@
+package ops
+
+import (
+ "database/sql"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// TestConfigGenSkipsHibernatingRoutes pins Eleven's two-replicators lesson in
+// executable form: a route with Hibernate=true must never appear in the fleet
+// litestream config, because the activator's embedded Litestream owns that
+// database's replication — a second replicator on the same WAL would fight it
+// for checkpointing.
+func TestConfigGenSkipsHibernatingRoutes(t *testing.T) {
+ dir := t.TempDir()
+ reg, err := registry.Open(filepath.Join(dir, "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ awake := filepath.Join(dir, "a.example.com.db")
+ asleep := filepath.Join(dir, "b.example.com.db")
+ for _, p := range []string{awake, asleep} {
+ if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := reg.Upsert(registry.Route{Host: "a.example.com", DBPath: awake}); err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.Upsert(registry.Route{Host: "b.example.com", DBPath: asleep, Hibernate: true}); err != nil {
+ t.Fatal(err)
+ }
+
+ dbs, err := FleetDBs(reg, dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cfg := LitestreamConfig(dbs, "bucket-x", "fleet", "eu-west-1")
+ if !strings.Contains(cfg, awake) {
+ t.Fatalf("awake route's DB missing from config:\n%s", cfg)
+ }
+ if strings.Contains(cfg, asleep) {
+ t.Fatalf("hibernating route's DB must be excluded (activator owns its replication):\n%s", cfg)
+ }
+ // The registry itself lives in the data dir and is not anyone's hibernating
+ // instance — it must be replicated.
+ if !strings.Contains(cfg, filepath.Join(dir, "registry.db")) {
+ t.Fatalf("registry.db missing from config:\n%s", cfg)
+ }
+ if !strings.Contains(cfg, "GENERATED") {
+ t.Fatalf("generated-file warning missing:\n%s", cfg)
+ }
+ if !strings.Contains(cfg, "region: eu-west-1") {
+ t.Fatalf("replica region missing:\n%s", cfg)
+ }
+}
+
+// The round-trip is the contract that lets restore-verify read the generated
+// config without a YAML parser: whatever LitestreamConfig writes,
+// ParseLitestreamDBs must recover exactly.
+func TestConfigRoundTrip(t *testing.T) {
+ in := []string{"/data/registry.db", "/data/a.example.com.db", "/data/b.example.com.db"}
+ got := ParseLitestreamDBs(LitestreamConfig(in, "b", "p", "eu-west-1"))
+ if len(got) != len(in) {
+ t.Fatalf("round-trip lost databases: in %d, out %d (%v)", len(in), len(got), got)
+ }
+ want := map[string]bool{}
+ for _, p := range in {
+ want[p] = true
+ }
+ for _, p := range got {
+ if !want[p] {
+ t.Fatalf("round-trip invented path %q", p)
+ }
+ }
+}
+
+func TestParseLitestreamDBsIgnoresNoise(t *testing.T) {
+ got := ParseLitestreamDBs("# comment\ndbs:\n - path: /data/x.db\n replicas:\n - url: s3://b/p/x.db\n region: eu-west-1\n\n")
+ if len(got) != 1 || got[0] != "/data/x.db" {
+ t.Fatalf("ParseLitestreamDBs = %#v, want [/data/x.db]", got)
+ }
+ if got := ParseLitestreamDBs(""); len(got) != 0 {
+ t.Fatalf("ParseLitestreamDBs(\"\") = %#v, want none", got)
+ }
+}
+
+// Re-running the generator over an unchanged fleet must produce a
+// byte-identical file — that is what lets the `litestream` verb report
+// "unchanged" and skip a restart-worthy config write.
+func TestConfigChangeDetection(t *testing.T) {
+ dbs := []string{"/data/b.db", "/data/a.db", "/data/registry.db"}
+ first := LitestreamConfig(dbs, "bucket", "prefix", "eu-west-1")
+ second := LitestreamConfig([]string{"/data/registry.db", "/data/a.db", "/data/b.db"}, "bucket", "prefix", "eu-west-1")
+ if first != second {
+ t.Fatalf("same fleet produced different configs:\n--- first\n%s\n--- second\n%s", first, second)
+ }
+}
+
+func TestCheckRestoredDBRejectsEmptySchema(t *testing.T) {
+ dir := t.TempDir()
+
+ good := filepath.Join(dir, "good.db")
+ db, err := sql.Open("sqlite", good)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := db.Exec("CREATE TABLE t (id INTEGER PRIMARY KEY); INSERT INTO t VALUES (1)"); err != nil {
+ t.Fatal(err)
+ }
+ db.Close()
+ if err := CheckRestoredDB(good); err != nil {
+ t.Fatalf("CheckRestoredDB(good) = %v, want nil", err)
+ }
+
+ // A schema-less database restores "successfully" but would be useless in a
+ // disaster — the check must refuse it.
+ empty := filepath.Join(dir, "empty.db")
+ db, err = sql.Open("sqlite", empty)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := db.Exec("PRAGMA user_version=1"); err != nil { // force file creation
+ t.Fatal(err)
+ }
+ db.Close()
+ if err := CheckRestoredDB(empty); err == nil {
+ t.Fatal("CheckRestoredDB accepted a database with no tables")
+ }
+
+ // Garbage bytes must fail loudly, not pass as an open-able file.
+ junk := filepath.Join(dir, "junk.db")
+ if err := os.WriteFile(junk, []byte("this is not a sqlite database at all"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := CheckRestoredDB(junk); err == nil {
+ t.Fatal("CheckRestoredDB accepted a non-SQLite file")
+ }
+}
diff --git a/internal/release/adopt.go b/internal/release/adopt.go
new file mode 100644
index 000000000..261879ec4
--- /dev/null
+++ b/internal/release/adopt.go
@@ -0,0 +1,217 @@
+// Adoption — one check-and-apply sweep over the registry, shaped after
+// Eleven updater.go's updatePass, simplified: channel pointers replace the
+// GitHub feed, the Store replaces HTTP downloads, and per-host update prefs
+// are the registry row itself (Channel, Hibernate).
+package release
+
+import (
+ "context"
+ "crypto/ed25519"
+ "errors"
+ "fmt"
+ "os"
+ "runtime"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// adoptPlatform picks which manifest artifact a box installs — a var so
+// tests can exercise resolution without faking runtime.GOOS/GOARCH.
+var adoptPlatform = runtime.GOOS + "-" + runtime.GOARCH
+
+// Pass is one full adoption sweep: group instance routes by (App, Channel),
+// resolve each channel pointer once, verify signatures and artifact sha256s,
+// download/stage missing versions, and repoint live/<host> symlinks.
+//
+// Die-free by design: per-host and per-channel problems log and continue —
+// an unreachable channel or one broken host must not strand the rest of the
+// fleet unchecked. Only foundation failures (registry unreadable, bake clock
+// unwritable) fail the pass.
+//
+// restart cycles an awake host's process onto the new symlink target (the
+// activator's proc converge); nil means symlink-only for every host. Hosts
+// whose route has Hibernate=true always get symlink-only and are NEVER
+// woken: an hourly wake would defeat hibernation, and the next wake execs
+// the new binary anyway since the spawn resolves live/<host> at exec time.
+// The trade (from Eleven): no health check for a sleeper — a bad build
+// surfaces at wake, recovered by Rollback, which repoints back. Hibernating
+// hosts are processed AFTER every awake host, so an awake host discovers a
+// boot-broken build (and poisons it) before any sleeper is moved onto it.
+func Pass(ctx context.Context, reg *registry.Registry, s Store, pub ed25519.PublicKey, optDir string, restart func(host string) error) error {
+ routes, err := reg.List()
+ if err != nil {
+ return err
+ }
+
+ // Resolve every distinct (App, Channel) once. Verification failures kill
+ // the group (nil entry), not the pass.
+ type groupKey struct{ app, channel string }
+ pointers := map[groupKey]*ChannelPointer{}
+ for _, rt := range routes {
+ if !adoptable(rt) {
+ continue
+ }
+ k := groupKey{rt.App, routeChannel(rt)}
+ if _, done := pointers[k]; done {
+ continue
+ }
+ ptr, err := Channel(ctx, s, pub, k.app, k.channel)
+ if err != nil {
+ pointers[k] = nil // resolved-and-failed: don't retry per host
+ if errors.Is(err, os.ErrNotExist) {
+ fmt.Printf(" %s/%s: channel has no release yet\n", k.app, k.channel)
+ } else {
+ fmt.Printf(" %s/%s: channel pointer rejected: %v\n", k.app, k.channel, err)
+ }
+ continue
+ }
+ pointers[k] = ptr
+ }
+
+ // Tick the bake clock: record every channel head this box hasn't seen at
+ // that head before, so the supply-chain window is measured from the box's
+ // own first sighting, never a feed timestamp (bake.go). The clock only
+ // ever ticks for pointers that verified above.
+ now := time.Now()
+ bake := LoadFirstSeen(FirstSeenPath(optDir))
+ changed := false
+ for k, ptr := range pointers {
+ if ptr != nil && bake.Observe(k.app, k.channel, ptr.Version, now) {
+ changed = true
+ }
+ }
+ if changed {
+ if err := bake.Save(); err != nil {
+ return fmt.Errorf("record bake clock: %v", err)
+ }
+ }
+
+ // A version that failed its restart on one host isn't tried on the next
+ // this pass — one bad build shouldn't bounce every host in turn.
+ poisoned := map[string]bool{}
+
+ // Awake hosts first, hibernating second: repoints have no health gate, so
+ // let an awake host discover (and poison) a broken build before any
+ // sleeper is moved onto it.
+ var hibernating []registry.Route
+ for _, rt := range routes {
+ if !adoptable(rt) {
+ continue
+ }
+ if rt.Hibernate {
+ hibernating = append(hibernating, rt)
+ continue
+ }
+ ptr := pointers[groupKey{rt.App, routeChannel(rt)}]
+ adoptHost(ctx, reg, s, optDir, bake, poisoned, rt, ptr, restart, now)
+ }
+ for _, rt := range hibernating {
+ ptr := pointers[groupKey{rt.App, routeChannel(rt)}]
+ // restart=nil: hibernating hosts get the pure repoint — never woken.
+ adoptHost(ctx, reg, s, optDir, bake, poisoned, rt, ptr, nil, now)
+ }
+ return nil
+}
+
+// adoptable filters the routes a Pass moves: lifecycle-managed instances of
+// a known app that are ready (an app-less or mid-provision route has nothing
+// to adopt or must not be disturbed).
+func adoptable(rt registry.Route) bool {
+ return rt.Kind == "instance" && rt.App != "" && (rt.Status == "" || rt.Status == "ready")
+}
+
+// routeChannel is the channel a route follows; "" means stable.
+func routeChannel(rt registry.Route) string {
+ if rt.Channel == "" {
+ return "stable"
+ }
+ return rt.Channel
+}
+
+// adoptHost converges one host onto its channel head ptr (nil = the group's
+// pointer didn't resolve; skip quietly, the group phase already logged).
+// restart==nil is the symlink-only path (hibernating hosts, or a caller with
+// no proc manager): repoint and record, never wake, never health-gate.
+func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir string, bake *FirstSeen, poisoned map[string]bool, rt registry.Route, ptr *ChannelPointer, restart func(host string) error, now time.Time) {
+ if ptr == nil {
+ return
+ }
+ channel := routeChannel(rt)
+ ver := ptr.Version
+ cur, err := LiveVersion(optDir, rt.Host)
+ if err != nil {
+ fmt.Printf(" %s: read live link: %v — skipped\n", rt.Host, err)
+ return
+ }
+ if cur == ver {
+ return // up to date — stay quiet on the periodic tick
+ }
+ if hold := Hold(bake, rt.App, channel, ver, DefaultHolds, false, now); hold != "" {
+ fmt.Printf(" %s: %s (%s) is %s\n", rt.Host, ver, channel, hold)
+ return
+ }
+ if poisoned[ver] {
+ fmt.Printf(" %s: skipping %s (failed on another host this pass)\n", rt.Host, ver)
+ return
+ }
+ if err := ensureStaged(ctx, s, optDir, rt.App, ver); err != nil {
+ fmt.Printf(" %s: %v\n", rt.Host, err)
+ return
+ }
+ if err := PointLive(optDir, rt.Host, ver, rt.App); err != nil {
+ fmt.Printf(" %s: repoint failed: %v\n", rt.Host, err)
+ return
+ }
+ if restart != nil {
+ if err := restart(rt.Host); err != nil {
+ // Poison the build for the rest of the pass and repoint back —
+ // the host keeps running what it ran.
+ poisoned[ver] = true
+ if cur != "" {
+ _ = PointLive(optDir, rt.Host, cur, rt.App)
+ }
+ fmt.Printf(" %s: update to %s failed: %v — rolled the symlink back\n", rt.Host, ver, err)
+ return
+ }
+ }
+ if err := reg.SetVersions(rt.Host, ver, cur); err != nil {
+ fmt.Printf(" %s: repointed but recording the version failed: %v\n", rt.Host, err)
+ return
+ }
+ if rt.Hibernate {
+ fmt.Printf(" %s: repointed %s -> %s (%s; hibernating — runs it on next wake)\n", rt.Host, cur, ver, channel)
+ } else {
+ fmt.Printf(" %s: updated %s -> %s (%s)\n", rt.Host, cur, ver, channel)
+ }
+}
+
+// ensureStaged makes ver's build of app available in the version store,
+// downloading and sha256-verifying it if needed. Idempotent across hosts in
+// one pass. The manifest supplies the sha; the bytes are refused unless they
+// hash to it — the channel pointer's signature plus this gate is the whole
+// trust chain from operator to disk.
+func ensureStaged(ctx context.Context, s Store, optDir, app, ver string) error {
+ if _, err := os.Stat(VersionPath(optDir, ver, app)); err == nil {
+ return nil
+ }
+ m, err := GetManifest(ctx, s, app, ver)
+ if err != nil {
+ return fmt.Errorf("manifest for %s/%s: %w", app, ver, err)
+ }
+ art, ok := m.Artifacts[adoptPlatform]
+ if !ok {
+ // A box facing a release with no build for its platform: skip, never
+ // install the wrong arch. The next release ships it and the box
+ // catches up.
+ return fmt.Errorf("release %s has no %s build — skipping", ver, adoptPlatform)
+ }
+ body, _, err := s.Get(ctx, ArtifactKey(app, ver, art.File))
+ if err != nil {
+ return fmt.Errorf("download %s: %w", ver, err)
+ }
+ if got := etagOf(body); got != art.SHA256 {
+ return fmt.Errorf("download %s: sha256 mismatch (refusing to stage)", ver)
+ }
+ return StageArtifact(optDir, ver, app, body)
+}
diff --git a/internal/release/adopt_test.go b/internal/release/adopt_test.go
new file mode 100644
index 000000000..4f10b9019
--- /dev/null
+++ b/internal/release/adopt_test.go
@@ -0,0 +1,193 @@
+package release
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+func testRegistry(t *testing.T) *registry.Registry {
+ t.Helper()
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatalf("open registry: %v", err)
+ }
+ return reg
+}
+
+func addRoute(t *testing.T, reg *registry.Registry, host, app, channel string, hibernate bool) {
+ t.Helper()
+ if err := reg.Upsert(registry.Route{
+ Host: host, Kind: "instance", App: app, Channel: channel, Hibernate: hibernate,
+ }); err != nil {
+ t.Fatalf("upsert %s: %v", host, err)
+ }
+}
+
+func TestPassRepointsAwakeAndHibernating(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ priv, pub := testKeys(t)
+ opt := t.TempDir()
+ reg := testRegistry(t)
+
+ // canary/* has no bake window, so the pass applies immediately.
+ shipRelease(t, s, "myapp", "v1", adoptPlatform, []byte("build one"))
+ if err := Promote(ctx, s, priv, "myapp", "canary/x", "v1"); err != nil {
+ t.Fatalf("Promote: %v", err)
+ }
+ addRoute(t, reg, "awake.example.com", "myapp", "canary/x", false)
+ addRoute(t, reg, "sleepy.example.com", "myapp", "canary/x", true)
+
+ var restarted []string
+ restart := func(host string) error { restarted = append(restarted, host); return nil }
+ if err := Pass(ctx, reg, s, pub, opt, restart); err != nil {
+ t.Fatalf("Pass: %v", err)
+ }
+
+ for _, host := range []string{"awake.example.com", "sleepy.example.com"} {
+ if v, _ := LiveVersion(opt, host); v != "v1" {
+ t.Fatalf("%s live = %q, want v1", host, v)
+ }
+ rt, err := reg.Get(host)
+ if err != nil || rt == nil || rt.Version != "v1" {
+ t.Fatalf("%s registry version = %+v, %v", host, rt, err)
+ }
+ }
+ // Only the awake host was restarted; the hibernating host is NEVER woken
+ // — its next wake execs the new binary via the live symlink.
+ if len(restarted) != 1 || restarted[0] != "awake.example.com" {
+ t.Fatalf("restarted = %v, want only awake.example.com", restarted)
+ }
+ // The staged artifact matches the shipped bytes (sha256-gated download).
+ body, err := os.ReadFile(VersionPath(opt, "v1", "myapp"))
+ if err != nil || string(body) != "build one" {
+ t.Fatalf("staged bytes = %q, %v", body, err)
+ }
+}
+
+func TestPassPoisonedBuildSkipsRemainingHosts(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ priv, pub := testKeys(t)
+ opt := t.TempDir()
+ reg := testRegistry(t)
+
+ // Both hosts start on v1 (staged locally), channel head moves to v2.
+ shipRelease(t, s, "myapp", "v1", adoptPlatform, []byte("good"))
+ shipRelease(t, s, "myapp", "v2", adoptPlatform, []byte("bad"))
+ if err := StageArtifact(opt, "v1", "myapp", []byte("good")); err != nil {
+ t.Fatal(err)
+ }
+ for _, h := range []string{"a.example.com", "b.example.com"} {
+ addRoute(t, reg, h, "myapp", "canary/x", false)
+ if err := PointLive(opt, h, "v1", "myapp"); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := Promote(ctx, s, priv, "myapp", "canary/x", "v2"); err != nil {
+ t.Fatalf("Promote v2: %v", err)
+ }
+
+ // v2 fails its restart on the first host — it must be poisoned for the
+ // rest of the pass, never cycling every host in turn.
+ var calls int
+ restart := func(host string) error { calls++; return errors.New("unit failed") }
+ if err := Pass(ctx, reg, s, pub, opt, restart); err != nil {
+ t.Fatalf("Pass: %v", err)
+ }
+ if calls != 1 {
+ t.Fatalf("restart called %d times, want 1 (poisoned build must not bounce every host)", calls)
+ }
+ // The failed host was rolled back to what it ran; the second never moved.
+ for _, h := range []string{"a.example.com", "b.example.com"} {
+ if v, _ := LiveVersion(opt, h); v != "v1" {
+ t.Fatalf("%s live = %q, want v1", h, v)
+ }
+ }
+}
+
+func TestPassStableHeldByBakeWindow(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ priv, pub := testKeys(t)
+ opt := t.TempDir()
+ reg := testRegistry(t)
+
+ shipRelease(t, s, "myapp", "v1", adoptPlatform, []byte("bin"))
+ if err := Promote(ctx, s, priv, "myapp", "stable", "v1"); err != nil {
+ t.Fatalf("Promote: %v", err)
+ }
+ addRoute(t, reg, "h.example.com", "myapp", "", false) // "" = stable
+
+ if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil {
+ t.Fatalf("Pass: %v", err)
+ }
+ // A brand-new stable head is inside its 72h supply-chain window: the
+ // clock started this pass, nothing installs.
+ if v, _ := LiveVersion(opt, "h.example.com"); v != "" {
+ t.Fatalf("live = %q, want unmoved (bake hold)", v)
+ }
+ // The pass persisted the first sighting so the window is measured from
+ // the box's own clock.
+ st := LoadFirstSeen(FirstSeenPath(opt))
+ if _, ok := st.EligibleAt("myapp", "stable", "v1", DefaultHolds); !ok {
+ t.Fatalf("Pass did not tick the first-seen clock")
+ }
+}
+
+func TestPassSkipsUnadoptableRoutes(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ priv, pub := testKeys(t)
+ opt := t.TempDir()
+ reg := testRegistry(t)
+
+ shipRelease(t, s, "myapp", "v1", adoptPlatform, []byte("bin"))
+ if err := Promote(ctx, s, priv, "myapp", "canary/x", "v1"); err != nil {
+ t.Fatalf("Promote: %v", err)
+ }
+ // A static route, an app-less instance, and a mid-provision instance:
+ // none may be moved by an adoption pass.
+ if err := reg.Upsert(registry.Route{Host: "site.example.com", Kind: "static", App: "myapp", Channel: "canary/x"}); err != nil {
+ t.Fatal(err)
+ }
+ addRoute(t, reg, "noapp.example.com", "", "canary/x", false)
+ if err := reg.Upsert(registry.Route{Host: "prov.example.com", Kind: "instance", App: "myapp", Channel: "canary/x", Status: "creating"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil {
+ t.Fatalf("Pass: %v", err)
+ }
+ for _, h := range []string{"site.example.com", "noapp.example.com", "prov.example.com"} {
+ if v, _ := LiveVersion(opt, h); v != "" {
+ t.Fatalf("%s was moved (%q) but is not adoptable", h, v)
+ }
+ }
+}
+
+func TestPassMissingPlatformArtifactSkips(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ priv, pub := testKeys(t)
+ opt := t.TempDir()
+ reg := testRegistry(t)
+
+ // Shipped for some other platform only: this box must skip, never
+ // install the wrong arch.
+ shipRelease(t, s, "myapp", "v1", "plan9-mips", []byte("bin"))
+ if err := Promote(ctx, s, priv, "myapp", "canary/x", "v1"); err != nil {
+ t.Fatalf("Promote: %v", err)
+ }
+ addRoute(t, reg, "h.example.com", "myapp", "canary/x", false)
+ if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil {
+ t.Fatalf("Pass: %v", err)
+ }
+ if v, _ := LiveVersion(opt, "h.example.com"); v != "" {
+ t.Fatalf("live = %q, want unmoved (no artifact for this platform)", v)
+ }
+}
diff --git a/internal/release/bake.go b/internal/release/bake.go
new file mode 100644
index 000000000..7928d9e7d
--- /dev/null
+++ b/internal/release/bake.go
@@ -0,0 +1,179 @@
+// Bake windows — ported from Eleven bake.go, intact.
+//
+// The supply-chain bake window: beta and stable subscribers don't install a
+// release the moment it appears — the box waits until the release has sat at
+// the channel head for a while first, so a compromised release that gets
+// caught and pulled back never reaches them. The clock is THIS BOX's own
+// observation ("when did I first see this version at this channel head"),
+// never a manifest timestamp: an attacker holding the signing key controls
+// every timestamp in the feed and would simply backdate one, but cannot
+// backdate what the box already saw.
+//
+// canary/* has no window — it IS the operator's own deploy path (`ship &&
+// promote`). beta's 24h absorbs the caught-within-a-day mass of real
+// supply-chain incidents; stable's 72h additionally spans a weekend, so a
+// Friday-night malicious publish is seen by a human on Monday with margin to
+// pull it back.
+//
+// There is deliberately NO publisher-side bypass ("critical fix, skip the
+// hold") — that flag is the first thing an attacker sets. The bypass is
+// local and explicit (the ignore parameter of Hold), which needs root on the
+// box.
+package release
+
+import (
+ "encoding/json"
+ "fmt"
+ "math/rand"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+)
+
+// DefaultHolds is the default bake window per channel; per-deployment
+// overrides come from host config as a replacement map passed by the caller.
+var DefaultHolds = map[string]time.Duration{
+ "canary/*": 0,
+ "beta": 24 * time.Hour,
+ "stable": 72 * time.Hour,
+}
+
+// HoldFor resolves a channel's bake window: an exact entry wins, then a
+// "<head>/*" wildcard for two-segment channels (canary/<branch>), and an
+// unknown channel gets the longest standard window — fail closed, never
+// zero, for a channel nobody configured.
+func HoldFor(holds map[string]time.Duration, channel string) time.Duration {
+ if d, ok := holds[channel]; ok {
+ return d
+ }
+ if i := strings.Index(channel, "/"); i >= 0 {
+ if d, ok := holds[channel[:i]+"/*"]; ok {
+ return d
+ }
+ }
+ return 72 * time.Hour
+}
+
+// FirstSeenPath is where a box persists its bake clock, under its opt dir.
+func FirstSeenPath(optDir string) string {
+ return filepath.Join(optDir, "channel-first-seen.json")
+}
+
+// firstSeenKeep bounds each channel's history. Past heads are kept (not just
+// the current one) so a pull-back to a previously-seen version is immediately
+// eligible — evacuating a bad release must never itself sit in a bake window.
+const firstSeenKeep = 12
+
+// FirstSeen is the bake clock: for each app/channel, when this box first
+// observed each version at its head.
+type FirstSeen struct {
+ // Jitter desynchronises a fleet of boxes: each waits hold*(1+Jitter), so
+ // boxes don't all install in the same hour and the earliest become
+ // canaries for the rest. Minted once per box, in [0, 0.25).
+ Jitter float64 `json:"jitter"`
+ // Channels is keyed "<app>/<channel>" -> version -> first-seen time.
+ Channels map[string]map[string]time.Time `json:"channels"`
+
+ path string // where Save persists; set by LoadFirstSeen
+}
+
+func channelClockKey(app, channel string) string { return app + "/" + channel }
+
+// LoadFirstSeen reads the bake clock from path. A missing or corrupt file is
+// an empty clock — every head then bakes from "now", which fails closed (a
+// wiped clock delays updates; it never lets one through early).
+func LoadFirstSeen(path string) *FirstSeen {
+ st := &FirstSeen{path: path}
+ if raw, err := os.ReadFile(path); err == nil {
+ _ = json.Unmarshal(raw, st)
+ }
+ if st.Channels == nil {
+ st.Channels = map[string]map[string]time.Time{}
+ }
+ if st.Jitter < 0 || st.Jitter >= 0.25 {
+ st.Jitter = 0
+ }
+ return st
+}
+
+// Observe records a channel head this box hasn't seen at that head before.
+// Returns whether anything changed (the caller persists via Save). Never
+// call it on a pointer that failed verification — the clock only ever ticks
+// for signed feeds. An already-known version's timestamp is NEVER touched:
+// that preserved first sighting is exactly what makes a pointer moved BACK
+// to a previously-seen version (rollback, evacuation) immediately eligible.
+func (st *FirstSeen) Observe(app, channel, version string, now time.Time) bool {
+ changed := false
+ if st.Jitter == 0 && len(st.Channels) == 0 {
+ // A fresh box mints its jitter once, alongside its first observation.
+ st.Jitter = rand.Float64() * 0.25
+ changed = true
+ }
+ key := channelClockKey(app, channel)
+ seen := st.Channels[key]
+ if seen == nil {
+ seen = map[string]time.Time{}
+ st.Channels[key] = seen
+ }
+ if _, ok := seen[version]; !ok {
+ seen[version] = now.UTC().Truncate(time.Second)
+ changed = true
+ }
+ // Prune oldest entries beyond the cap — the just-recorded head is the
+ // newest, so it always survives.
+ for len(seen) > firstSeenKeep {
+ oldest, at := "", time.Time{}
+ for v, t := range seen {
+ if oldest == "" || t.Before(at) {
+ oldest, at = v, t
+ }
+ }
+ delete(seen, oldest)
+ }
+ return changed
+}
+
+// Save persists the clock back to the path it was loaded from.
+func (st *FirstSeen) Save() error {
+ raw, err := json.MarshalIndent(st, "", " ")
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(st.path, append(raw, '\n'), 0o644)
+}
+
+// EligibleAt is the moment a version first seen at a channel head may be
+// installed: firstSeen + hold*(1+jitter). ok=false when the version has no
+// record at this channel's head — brand new, so not eligible (callers that
+// Observe first make that unreachable; a status read that doesn't tick the
+// clock can legitimately see it).
+func (st *FirstSeen) EligibleAt(app, channel, version string, holds map[string]time.Duration) (time.Time, bool) {
+ seen, ok := st.Channels[channelClockKey(app, channel)][version]
+ if !ok {
+ return time.Time{}, false
+ }
+ d := HoldFor(holds, channel)
+ if d == 0 {
+ return seen, true
+ }
+ return seen.Add(time.Duration(float64(d) * (1 + st.Jitter))), true
+}
+
+// Hold returns a non-empty human reason while a channel head must not yet be
+// installed on this box, "" once it may be. ignore is the local, root-only
+// override — deliberately never a publisher-side flag.
+func Hold(st *FirstSeen, app, channel, version string, holds map[string]time.Duration, ignore bool, now time.Time) string {
+ if ignore {
+ return ""
+ }
+ at, ok := st.EligibleAt(app, channel, version, holds)
+ if !ok {
+ // Fail closed: no record means no clock has run, so nothing installs.
+ return "in its supply-chain bake window (clock starts this pass)"
+ }
+ if now.Before(at) {
+ return fmt.Sprintf("in its supply-chain bake window until %s", at.UTC().Format("2006-01-02 15:04 MST"))
+ }
+ return ""
+}
diff --git a/internal/release/bake_test.go b/internal/release/bake_test.go
new file mode 100644
index 000000000..c1743710d
--- /dev/null
+++ b/internal/release/bake_test.go
@@ -0,0 +1,171 @@
+package release
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+// Jitter is in [0, 0.25), so a hold of H is eligible somewhere in
+// [firstSeen+H, firstSeen+1.25H). 91h is safely past stable's worst case
+// (72h * 1.25 = 90h); 71h is safely inside the best case.
+const (
+ insideStableHold = 71 * time.Hour
+ pastStableHold = 91 * time.Hour
+)
+
+func TestBakeHoldBlocksNewStableUntilHeld(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "first-seen.json")
+ st := LoadFirstSeen(path)
+ t0 := time.Now()
+ if !st.Observe("myapp", "stable", "v1", t0) {
+ t.Fatalf("first Observe reported no change")
+ }
+ if hold := Hold(st, "myapp", "stable", "v1", DefaultHolds, false, t0); hold == "" {
+ t.Fatalf("brand-new stable head not held")
+ }
+ if hold := Hold(st, "myapp", "stable", "v1", DefaultHolds, false, t0.Add(insideStableHold)); hold == "" {
+ t.Fatalf("stable head released before its 72h window")
+ }
+ if hold := Hold(st, "myapp", "stable", "v1", DefaultHolds, false, t0.Add(pastStableHold)); hold != "" {
+ t.Fatalf("stable head still held past its window: %s", hold)
+ }
+ // A version with NO first-seen record fails closed.
+ if hold := Hold(st, "myapp", "stable", "vGhost", DefaultHolds, false, t0.Add(pastStableHold)); hold == "" {
+ t.Fatalf("unseen version not held (must fail closed)")
+ }
+ // The local root-only override releases immediately.
+ if hold := Hold(st, "myapp", "stable", "v1", DefaultHolds, true, t0); hold != "" {
+ t.Fatalf("ignore=true still held: %s", hold)
+ }
+ // canary/* has no window: eligible the moment it is first seen.
+ st.Observe("myapp", "canary/x", "v1", t0)
+ if hold := Hold(st, "myapp", "canary/x", "v1", DefaultHolds, false, t0); hold != "" {
+ t.Fatalf("canary held: %s", hold)
+ }
+}
+
+// TestRollbackNeverBakes is THE key property: a pointer moved BACK to a
+// previously-seen version applies immediately — its first-seen record
+// predates the hold (that's how it got installed the first time), and
+// Observe never resets an existing timestamp. Evacuating a bad release must
+// never itself sit in a bake window.
+func TestRollbackNeverBakes(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ priv, pub := testKeys(t)
+ shipRelease(t, s, "myapp", "v1", "linux-amd64", []byte("one"))
+ shipRelease(t, s, "myapp", "v2", "linux-amd64", []byte("two"))
+
+ st := LoadFirstSeen(filepath.Join(t.TempDir(), "first-seen.json"))
+ t0 := time.Now()
+
+ // v1 promoted to stable; the box first sees it at t0, bakes, installs.
+ if err := Promote(ctx, s, priv, "myapp", "stable", "v1"); err != nil {
+ t.Fatalf("Promote v1: %v", err)
+ }
+ ptr, err := Channel(ctx, s, pub, "myapp", "stable")
+ if err != nil {
+ t.Fatalf("Channel: %v", err)
+ }
+ st.Observe("myapp", "stable", ptr.Version, t0)
+
+ // Time passes; v1's window elapses; v2 is promoted and first seen NOW.
+ t1 := t0.Add(pastStableHold)
+ if err := Promote(ctx, s, priv, "myapp", "stable", "v2"); err != nil {
+ t.Fatalf("Promote v2: %v", err)
+ }
+ ptr, _ = Channel(ctx, s, pub, "myapp", "stable")
+ st.Observe("myapp", "stable", ptr.Version, t1)
+ if hold := Hold(st, "myapp", "stable", ptr.Version, DefaultHolds, false, t1); hold == "" {
+ t.Fatalf("fresh v2 not held")
+ }
+
+ // v2 turns out bad — rollback. The pointer is back at v1, and v1 applies
+ // IMMEDIATELY: its first-seen record is t0, long past the hold. Observe
+ // must not reset it.
+ if err := Rollback(ctx, s, priv, "myapp", "stable"); err != nil {
+ t.Fatalf("Rollback: %v", err)
+ }
+ ptr, err = Channel(ctx, s, pub, "myapp", "stable")
+ if err != nil {
+ t.Fatalf("Channel after rollback: %v", err)
+ }
+ if ptr.Version != "v1" {
+ t.Fatalf("rollback pointer = %s, want v1", ptr.Version)
+ }
+ st.Observe("myapp", "stable", ptr.Version, t1) // re-seen at head: no reset
+ if hold := Hold(st, "myapp", "stable", ptr.Version, DefaultHolds, false, t1); hold != "" {
+ t.Fatalf("rolled-back v1 held: %s (evacuation must never bake)", hold)
+ }
+ // And v2 stays held — the rollback didn't launder it.
+ if hold := Hold(st, "myapp", "stable", "v2", DefaultHolds, false, t1); hold == "" {
+ t.Fatalf("bad v2 no longer held after rollback")
+ }
+}
+
+func TestLoadFirstSeenCorruptFailsClosed(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "first-seen.json")
+ if err := os.WriteFile(path, []byte("{ this is not json"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ st := LoadFirstSeen(path)
+ // A corrupt file is an empty clock: everything bakes from now — a wiped
+ // clock delays updates, it never lets one through early.
+ if hold := Hold(st, "myapp", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" {
+ t.Fatalf("corrupt clock did not fail closed")
+ }
+ // Missing file: same.
+ st2 := LoadFirstSeen(filepath.Join(t.TempDir(), "absent.json"))
+ if hold := Hold(st2, "myapp", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" {
+ t.Fatalf("missing clock did not fail closed")
+ }
+}
+
+func TestFirstSeenSaveLoadRoundTrip(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "first-seen.json")
+ st := LoadFirstSeen(path)
+ t0 := time.Now()
+ st.Observe("myapp", "stable", "v1", t0)
+ if st.Jitter < 0 || st.Jitter >= 0.25 {
+ t.Fatalf("minted jitter %v outside [0, 0.25)", st.Jitter)
+ }
+ if err := st.Save(); err != nil {
+ t.Fatalf("Save: %v", err)
+ }
+ st2 := LoadFirstSeen(path)
+ if st2.Jitter != st.Jitter {
+ t.Fatalf("jitter not persisted: %v != %v", st2.Jitter, st.Jitter)
+ }
+ at, ok := st2.EligibleAt("myapp", "stable", "v1", DefaultHolds)
+ if !ok {
+ t.Fatalf("first-seen record lost on reload")
+ }
+ want := t0.UTC().Truncate(time.Second).Add(time.Duration(float64(72*time.Hour) * (1 + st.Jitter)))
+ if !at.Equal(want) {
+ t.Fatalf("EligibleAt = %v, want %v", at, want)
+ }
+}
+
+func TestHoldForWildcardAndUnknown(t *testing.T) {
+ if d := HoldFor(DefaultHolds, "canary/any-branch"); d != 0 {
+ t.Fatalf("canary/* = %v, want 0", d)
+ }
+ if d := HoldFor(DefaultHolds, "beta"); d != 24*time.Hour {
+ t.Fatalf("beta = %v, want 24h", d)
+ }
+ if d := HoldFor(DefaultHolds, "stable"); d != 72*time.Hour {
+ t.Fatalf("stable = %v, want 72h", d)
+ }
+ // An unknown channel fails closed to the longest standard window.
+ if d := HoldFor(DefaultHolds, "mystery"); d != 72*time.Hour {
+ t.Fatalf("unknown channel = %v, want 72h (fail closed)", d)
+ }
+ // A per-deployment override map replaces the defaults wholesale.
+ custom := map[string]time.Duration{"stable": time.Hour}
+ if d := HoldFor(custom, "stable"); d != time.Hour {
+ t.Fatalf("override stable = %v, want 1h", d)
+ }
+}
diff --git a/internal/release/release.go b/internal/release/release.go
new file mode 100644
index 000000000..34e58cc7d
--- /dev/null
+++ b/internal/release/release.go
@@ -0,0 +1,477 @@
+// Package release is the release plane: content-addressed release manifests
+// and signed, CAS-guarded channel pointers in a deployment bucket.
+//
+// Ported from Eleven release.go (the signedManifest wrapper, key parsing,
+// keygen) with the trust split the plan calls for: release manifests are
+// content-addressed and immutable (their bucket path is claimed create-only,
+// so their integrity is their existence), while CHANNEL POINTERS are the
+// trust boundary — a pointer is what tells a fleet which build to run, so it
+// carries the Ed25519 signature and moves only via compare-and-swap.
+//
+// Bucket layout (deployment bucket):
+//
+// apps/<app>/releases/<version>/manifest.json content-addressed, immutable
+// apps/<app>/releases/<version>/<artifact files> binaries or static tree tar
+// apps/<app>/channels/<channel>.json SIGNED channel pointer, CAS via If-Match
+//
+// Keys: the private half is the CARLOS_RELEASE_KEY Ed25519 seed (env only —
+// secret manager -> shell variable -> process memory, never disk); the
+// public key is pinned in the deployment's host config
+// (CARLOS_RELEASE_PUBKEY), not baked into the binary.
+package release
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "regexp"
+ "strings"
+ "time"
+)
+
+// Manifest describes one shipped release of one app. It is content-addressed
+// (immutable at its bucket path), so it needs no signature of its own: a
+// verified channel pointer names its version, and the manifest's artifact
+// sha256s gate every byte downloaded from it.
+type Manifest struct {
+ Schema int `json:"schema"` // 1
+ App string `json:"app"`
+ Version string `json:"version"` // git short sha
+ Kind string `json:"kind"` // "binary" | "static"
+ Commit string `json:"commit,omitempty"`
+ BuiltBy string `json:"built_by,omitempty"`
+ BuiltAt string `json:"built_at,omitempty"`
+ // Artifacts is keyed "linux-arm64" etc for binaries, or "site" for a
+ // static tree tar.
+ Artifacts map[string]Artifact `json:"artifacts"`
+}
+
+// Artifact is one file of a release: its bucket file name and the sha256
+// every downloader must verify before staging it.
+type Artifact struct {
+ File string `json:"file"`
+ SHA256 string `json:"sha256"`
+ Size int64 `json:"size"`
+}
+
+// ChannelPointer says which version a channel currently serves. This is the
+// document the fleet trusts, so it ships wrapped in signedWrapper and only
+// moves via CAS.
+type ChannelPointer struct {
+ Schema int `json:"schema"`
+ App string `json:"app"`
+ Channel string `json:"channel"`
+ Version string `json:"version"`
+ PromotedAt time.Time `json:"promoted_at"`
+ // History is the previous versions, newest first, max historyMax —
+ // rollback targets. Rollback swaps Version <-> History[0], so a second
+ // rollback returns to where you were (Eleven versions.go's
+ // current<->previous swap, generalised).
+ History []string `json:"history,omitempty"`
+}
+
+// historyMax bounds a pointer's rollback history.
+const historyMax = 10
+
+// Bucket keys. Channel names may contain one "/" (canary/<branch>), which
+// simply nests the pointer one level deeper.
+func ManifestKey(app, version string) string {
+ return "apps/" + app + "/releases/" + version + "/manifest.json"
+}
+func ArtifactKey(app, version, file string) string {
+ return "apps/" + app + "/releases/" + version + "/" + file
+}
+func ChannelKey(app, channel string) string {
+ return "apps/" + app + "/channels/" + channel + ".json"
+}
+
+// appRe pins app names to the boring lowercase words the box maps to
+// /opt/carlos paths — anything fancier is a manifest bug, not a feature.
+// (Eleven release.go's serviceNameRe.)
+var appRe = regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`)
+
+func ValidApp(a string) bool { return appRe.MatchString(a) }
+
+// channelSegRe is one channel path segment.
+var channelSegRe = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`)
+
+// ValidChannel accepts "stable", "beta", any other single name, or a
+// two-segment "canary/<branch>" — each segment directory-name safe (the
+// channel lands in a bucket key and in on-box state files).
+func ValidChannel(c string) bool {
+ parts := strings.Split(c, "/")
+ if len(parts) == 0 || len(parts) > 2 {
+ return false
+ }
+ for _, p := range parts {
+ if p == "" || p == "." || p == ".." || len(p) > 64 || !channelSegRe.MatchString(p) {
+ return false
+ }
+ }
+ return true
+}
+
+// fileRe keeps artifact file names usable as single path segments.
+var fileRe = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`)
+
+func validArtifactFile(f string) bool {
+ return f != "" && f != "." && f != ".." && len(f) <= 128 && fileRe.MatchString(f)
+}
+
+// validateManifest rejects a manifest that parsed but can't be acted on.
+func validateManifest(m *Manifest) error {
+ if m.Schema != 1 {
+ return fmt.Errorf("unsupported manifest schema %d", m.Schema)
+ }
+ if !ValidApp(m.App) {
+ return fmt.Errorf("invalid app %q", m.App)
+ }
+ if !ValidVersion(m.Version) || m.Version == "dev" {
+ return fmt.Errorf("invalid version %q", m.Version)
+ }
+ if m.Kind != "binary" && m.Kind != "static" {
+ return fmt.Errorf("invalid kind %q (binary | static)", m.Kind)
+ }
+ if len(m.Artifacts) == 0 {
+ return fmt.Errorf("manifest has no artifacts")
+ }
+ for k, a := range m.Artifacts {
+ if !validArtifactFile(a.File) {
+ return fmt.Errorf("artifact %s: invalid file name %q", k, a.File)
+ }
+ if raw, err := hex.DecodeString(a.SHA256); err != nil || len(raw) != sha256.Size {
+ return fmt.Errorf("artifact %s: sha256 must be %d hex chars", k, sha256.Size*2)
+ }
+ if a.Size < 0 {
+ return fmt.Errorf("artifact %s: negative size", k)
+ }
+ }
+ return nil
+}
+
+// ---- signing ----------------------------------------------------------------
+
+// signedWrapper is the wire format of a channel pointer: the signature
+// covers the exact base64 payload string, so verification never depends on
+// re-marshalling JSON — verify first, decode after, no JSON canonicalisation
+// to get wrong. (Ported from Eleven release.go's signedManifest.)
+type signedWrapper struct {
+ Payload string `json:"payload"`
+ Sig string `json:"sig"`
+}
+
+// signPointer stamps PromotedAt (now) and wraps the pointer in the signed
+// wire format.
+func signPointer(priv ed25519.PrivateKey, p *ChannelPointer) ([]byte, error) {
+ p.Schema = 1
+ p.PromotedAt = time.Now().UTC().Truncate(time.Second)
+ if err := validatePointer(p, p.App, p.Channel); err != nil {
+ return nil, err
+ }
+ payload, err := json.MarshalIndent(p, "", " ")
+ if err != nil {
+ return nil, err
+ }
+ b64 := base64.StdEncoding.EncodeToString(payload)
+ sig := ed25519.Sign(priv, []byte(b64))
+ return json.MarshalIndent(signedWrapper{
+ Payload: b64,
+ Sig: base64.StdEncoding.EncodeToString(sig),
+ }, "", " ")
+}
+
+// verifyPointer checks the signature against pub and only then decodes and
+// validates the payload — everything the caller acts on came from under the
+// signature. wantApp/wantChannel are matched against the SIGNED payload, so
+// a validly-signed pointer copied from another channel's key (a cross-channel
+// replay) is rejected.
+func verifyPointer(pub ed25519.PublicKey, wrapper []byte, wantApp, wantChannel string) (*ChannelPointer, error) {
+ var sw signedWrapper
+ if err := json.Unmarshal(wrapper, &sw); err != nil {
+ return nil, fmt.Errorf("pointer wrapper: %v", err)
+ }
+ sig, err := base64.StdEncoding.DecodeString(sw.Sig)
+ if err != nil {
+ return nil, fmt.Errorf("pointer sig: %v", err)
+ }
+ if !ed25519.Verify(pub, []byte(sw.Payload), sig) {
+ return nil, fmt.Errorf("channel pointer signature verification failed")
+ }
+ payload, err := base64.StdEncoding.DecodeString(sw.Payload)
+ if err != nil {
+ return nil, fmt.Errorf("pointer payload: %v", err)
+ }
+ var p ChannelPointer
+ if err := json.Unmarshal(payload, &p); err != nil {
+ return nil, fmt.Errorf("pointer payload: %v", err)
+ }
+ if err := validatePointer(&p, wantApp, wantChannel); err != nil {
+ return nil, err
+ }
+ return &p, nil
+}
+
+func validatePointer(p *ChannelPointer, wantApp, wantChannel string) error {
+ if p.Schema != 1 {
+ return fmt.Errorf("unsupported pointer schema %d", p.Schema)
+ }
+ if p.App != wantApp || p.Channel != wantChannel {
+ return fmt.Errorf("pointer names %s/%s, expected %s/%s", p.App, p.Channel, wantApp, wantChannel)
+ }
+ if !ValidApp(p.App) || !ValidChannel(p.Channel) {
+ return fmt.Errorf("pointer has invalid app/channel %s/%s", p.App, p.Channel)
+ }
+ if !ValidVersion(p.Version) || p.Version == "dev" {
+ return fmt.Errorf("pointer has invalid version %q", p.Version)
+ }
+ if p.PromotedAt.IsZero() {
+ return fmt.Errorf("pointer has no promoted_at")
+ }
+ if len(p.History) > historyMax {
+ return fmt.Errorf("pointer history too long (%d > %d)", len(p.History), historyMax)
+ }
+ for _, v := range p.History {
+ if !ValidVersion(v) {
+ return fmt.Errorf("pointer history has invalid version %q", v)
+ }
+ }
+ return nil
+}
+
+// ---- keys -------------------------------------------------------------------
+
+// GenerateKey mints a fresh signing pair: the base64 seed (the operator's
+// secret — password manager, then CARLOS_RELEASE_KEY) and the base64 public
+// key (pinned in host config as CARLOS_RELEASE_PUBKEY).
+func GenerateKey() (seedB64, pubB64 string, err error) {
+ pub, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ return "", "", err
+ }
+ return base64.StdEncoding.EncodeToString(priv.Seed()),
+ base64.StdEncoding.EncodeToString(pub), nil
+}
+
+// ParseSigningKey turns a base64 Ed25519 seed — from CARLOS_RELEASE_KEY —
+// into the signing key. src only names the origin in the error.
+func ParseSigningKey(b64, src string) (ed25519.PrivateKey, error) {
+ seed, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64))
+ if err != nil || len(seed) != ed25519.SeedSize {
+ return nil, fmt.Errorf("%s: expected a base64 %d-byte Ed25519 seed", src, ed25519.SeedSize)
+ }
+ return ed25519.NewKeyFromSeed(seed), nil
+}
+
+// ParsePublicKey turns a base64 public key — from CARLOS_RELEASE_PUBKEY —
+// into the verification key.
+func ParsePublicKey(b64, src string) (ed25519.PublicKey, error) {
+ raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64))
+ if err != nil || len(raw) != ed25519.PublicKeySize {
+ return nil, fmt.Errorf("%s: expected a base64 %d-byte Ed25519 public key", src, ed25519.PublicKeySize)
+ }
+ return ed25519.PublicKey(raw), nil
+}
+
+// ---- top-level API ----------------------------------------------------------
+
+// Ship uploads a release: artifacts first, manifest LAST — the manifest is
+// the commit record, so a crashed ship leaves orphan artifact bytes, never a
+// manifest naming files that don't exist. Everything is create-only
+// (immutable): re-shipping identical bytes is an idempotent no-op, while
+// re-shipping DIFFERENT content under an existing version is refused — a
+// version id, once published, can never quietly mean something else.
+//
+// files is keyed by artifact map key ("linux-arm64", "site", ...). Ship
+// fills each Artifact's SHA256/Size from the bytes (and, when already set,
+// requires them to match); an empty Artifact.File defaults to the map key.
+func Ship(ctx context.Context, s Store, m Manifest, files map[string][]byte) error {
+ m.Schema = 1
+ for k, a := range m.Artifacts {
+ b, ok := files[k]
+ if !ok {
+ return fmt.Errorf("ship: manifest artifact %q has no file bytes", k)
+ }
+ sum := etagOf(b) // hex sha256
+ if a.SHA256 != "" && a.SHA256 != sum {
+ return fmt.Errorf("ship: artifact %q sha256 mismatch (manifest %s, bytes %s)", k, a.SHA256, sum)
+ }
+ a.SHA256 = sum
+ a.Size = int64(len(b))
+ if a.File == "" {
+ a.File = k
+ }
+ m.Artifacts[k] = a
+ }
+ for k := range files {
+ if _, ok := m.Artifacts[k]; !ok {
+ return fmt.Errorf("ship: file %q has no manifest artifact entry", k)
+ }
+ }
+ if err := validateManifest(&m); err != nil {
+ return fmt.Errorf("ship: %w", err)
+ }
+ for k, a := range m.Artifacts {
+ if err := putImmutable(ctx, s, ArtifactKey(m.App, m.Version, a.File), files[k]); err != nil {
+ return fmt.Errorf("ship: artifact %s: %w", k, err)
+ }
+ }
+ raw, err := json.MarshalIndent(m, "", " ")
+ if err != nil {
+ return err
+ }
+ if err := putImmutable(ctx, s, ManifestKey(m.App, m.Version), raw); err != nil {
+ return fmt.Errorf("ship: manifest: %w", err)
+ }
+ return nil
+}
+
+// putImmutable claims a content-addressed key create-only; a lost race is
+// fine iff the existing bytes are identical (idempotent re-ship), and an
+// error otherwise (an immutable path must never change meaning).
+func putImmutable(ctx context.Context, s Store, key string, body []byte) error {
+ _, err := s.Put(ctx, key, body, "")
+ if !errors.Is(err, ErrCASConflict) {
+ return err
+ }
+ cur, _, gerr := s.Get(ctx, key)
+ if gerr != nil {
+ return fmt.Errorf("%s exists but is unreadable: %w", key, gerr)
+ }
+ if !bytes.Equal(cur, body) {
+ return fmt.Errorf("%s already exists with different content — refusing to overwrite an immutable release object", key)
+ }
+ return nil
+}
+
+// GetManifest fetches and validates a release manifest, requiring it to name
+// the app/version its key claims. Not-found is os.ErrNotExist-wrapped.
+func GetManifest(ctx context.Context, s Store, app, version string) (*Manifest, error) {
+ raw, _, err := s.Get(ctx, ManifestKey(app, version))
+ if err != nil {
+ return nil, err
+ }
+ var m Manifest
+ if err := json.Unmarshal(raw, &m); err != nil {
+ return nil, fmt.Errorf("manifest %s/%s: %v", app, version, err)
+ }
+ if err := validateManifest(&m); err != nil {
+ return nil, fmt.Errorf("manifest %s/%s: %w", app, version, err)
+ }
+ if m.App != app || m.Version != version {
+ return nil, fmt.Errorf("manifest at %s names %s/%s", ManifestKey(app, version), m.App, m.Version)
+ }
+ return &m, nil
+}
+
+// Promote points a channel at a shipped version, signing the new pointer and
+// swapping it in via CAS (a concurrent promote surfaces as ErrCASConflict —
+// re-read and retry, never overwrite blind). The outgoing version is
+// prepended to History (capped at historyMax), which is what Rollback
+// consumes. An existing pointer must verify against this key's public half
+// before it is replaced: history from a document we can't trust doesn't get
+// carried forward.
+func Promote(ctx context.Context, s Store, priv ed25519.PrivateKey, app, channel, version string) error {
+ if !ValidApp(app) {
+ return fmt.Errorf("promote: invalid app %q", app)
+ }
+ if !ValidChannel(channel) {
+ return fmt.Errorf("promote: invalid channel %q", channel)
+ }
+ if !ValidVersion(version) || version == "dev" {
+ return fmt.Errorf("promote: invalid version %q", version)
+ }
+ // A channel only ever points at a version that was actually shipped.
+ if _, err := GetManifest(ctx, s, app, version); err != nil {
+ return fmt.Errorf("promote: version %s is not shipped: %w", version, err)
+ }
+ pub, _ := priv.Public().(ed25519.PublicKey)
+ key := ChannelKey(app, channel)
+ cur, etag, err := s.Get(ctx, key)
+ var old *ChannelPointer
+ switch {
+ case err == nil:
+ old, err = verifyPointer(pub, cur, app, channel)
+ if err != nil {
+ return fmt.Errorf("promote: existing pointer at %s: %w", key, err)
+ }
+ case errors.Is(err, os.ErrNotExist):
+ old, etag = &ChannelPointer{}, "" // first promote: create-only Put
+ default:
+ return err
+ }
+ next := &ChannelPointer{App: app, Channel: channel, Version: version, History: old.History}
+ if old.Version != "" && old.Version != version {
+ next.History = prependCapped(old.Version, old.History, historyMax)
+ }
+ wrapper, err := signPointer(priv, next)
+ if err != nil {
+ return err
+ }
+ _, err = s.Put(ctx, key, wrapper, etag)
+ return err
+}
+
+// Channel resolves a channel's current pointer, verifying its signature —
+// the one read path the fleet uses, so nothing unsigned ever reaches a box.
+// Not-found is os.ErrNotExist-wrapped.
+func Channel(ctx context.Context, s Store, pub ed25519.PublicKey, app, channel string) (*ChannelPointer, error) {
+ raw, _, err := s.Get(ctx, ChannelKey(app, channel))
+ if err != nil {
+ return nil, err
+ }
+ return verifyPointer(pub, raw, app, channel)
+}
+
+// Rollback swaps a channel back to History[0], pushing the outgoing version
+// onto History — so a second rollback returns to where you were. Boxes apply
+// it immediately per the bake rules: a box that has previously seen the
+// target at this channel's head has a first-seen record older than the hold
+// (that's how it got installed the first time), so evacuating a bad release
+// never itself sits in a bake window (bake.go).
+func Rollback(ctx context.Context, s Store, priv ed25519.PrivateKey, app, channel string) error {
+ if !ValidApp(app) || !ValidChannel(channel) {
+ return fmt.Errorf("rollback: invalid app/channel %s/%s", app, channel)
+ }
+ pub, _ := priv.Public().(ed25519.PublicKey)
+ key := ChannelKey(app, channel)
+ cur, etag, err := s.Get(ctx, key)
+ if err != nil {
+ return fmt.Errorf("rollback: %w", err)
+ }
+ old, err := verifyPointer(pub, cur, app, channel)
+ if err != nil {
+ return fmt.Errorf("rollback: existing pointer at %s: %w", key, err)
+ }
+ if len(old.History) == 0 {
+ return fmt.Errorf("rollback: %s/%s has no previous version recorded", app, channel)
+ }
+ next := &ChannelPointer{
+ App: app,
+ Channel: channel,
+ Version: old.History[0],
+ History: prependCapped(old.Version, old.History[1:], historyMax),
+ }
+ wrapper, err := signPointer(priv, next)
+ if err != nil {
+ return err
+ }
+ _, err = s.Put(ctx, key, wrapper, etag)
+ return err
+}
+
+// prependCapped puts v at the head of hist, dropping the tail beyond max.
+func prependCapped(v string, hist []string, max int) []string {
+ out := append([]string{v}, hist...)
+ if len(out) > max {
+ out = out[:max]
+ }
+ return out
+}
diff --git a/internal/release/release_test.go b/internal/release/release_test.go
new file mode 100644
index 000000000..e358a422d
--- /dev/null
+++ b/internal/release/release_test.go
@@ -0,0 +1,279 @@
+package release
+
+import (
+ "context"
+ "crypto/ed25519"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "testing"
+ "time"
+)
+
+// testKeys mints a signing pair through the exported key helpers, so the
+// round trip seed -> base64 -> key is exercised too.
+func testKeys(t *testing.T) (ed25519.PrivateKey, ed25519.PublicKey) {
+ t.Helper()
+ seedB64, pubB64, err := GenerateKey()
+ if err != nil {
+ t.Fatalf("GenerateKey: %v", err)
+ }
+ priv, err := ParseSigningKey(seedB64, "test")
+ if err != nil {
+ t.Fatalf("ParseSigningKey: %v", err)
+ }
+ pub, err := ParsePublicKey(pubB64, "test")
+ if err != nil {
+ t.Fatalf("ParsePublicKey: %v", err)
+ }
+ return priv, pub
+}
+
+// shipRelease ships a one-artifact binary release keyed by plat.
+func shipRelease(t *testing.T, s Store, app, version, plat string, body []byte) {
+ t.Helper()
+ m := Manifest{
+ App: app,
+ Version: version,
+ Kind: "binary",
+ Artifacts: map[string]Artifact{plat: {}},
+ }
+ if err := Ship(context.Background(), s, m, map[string][]byte{plat: body}); err != nil {
+ t.Fatalf("Ship %s/%s: %v", app, version, err)
+ }
+}
+
+func TestShipThenPromoteThenChannelRoundTrip(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ priv, pub := testKeys(t)
+
+ shipRelease(t, s, "myapp", "v1abc", "linux-amd64", []byte("binary bytes v1"))
+ if err := Promote(ctx, s, priv, "myapp", "canary/x", "v1abc"); err != nil {
+ t.Fatalf("Promote: %v", err)
+ }
+ ptr, err := Channel(ctx, s, pub, "myapp", "canary/x")
+ if err != nil {
+ t.Fatalf("Channel: %v", err)
+ }
+ if ptr.App != "myapp" || ptr.Channel != "canary/x" || ptr.Version != "v1abc" {
+ t.Fatalf("pointer = %+v", ptr)
+ }
+ if ptr.PromotedAt.IsZero() || len(ptr.History) != 0 {
+ t.Fatalf("pointer = %+v, want stamped PromotedAt and empty history", ptr)
+ }
+ // The manifest round-trips with the sha256 Ship computed.
+ m, err := GetManifest(ctx, s, "myapp", "v1abc")
+ if err != nil {
+ t.Fatalf("GetManifest: %v", err)
+ }
+ art := m.Artifacts["linux-amd64"]
+ if art.SHA256 != etagOf([]byte("binary bytes v1")) || art.Size != int64(len("binary bytes v1")) {
+ t.Fatalf("artifact = %+v", art)
+ }
+ body, _, err := s.Get(ctx, ArtifactKey("myapp", "v1abc", art.File))
+ if err != nil || string(body) != "binary bytes v1" {
+ t.Fatalf("artifact bytes = %q, %v", body, err)
+ }
+}
+
+func TestShipImmutable(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ // Re-shipping identical bytes is an idempotent no-op...
+ shipRelease(t, s, "myapp", "v1abc", "linux-amd64", []byte("same"))
+ shipRelease(t, s, "myapp", "v1abc", "linux-amd64", []byte("same"))
+ // ...but the same version with DIFFERENT content is refused.
+ m := Manifest{App: "myapp", Version: "v1abc", Kind: "binary",
+ Artifacts: map[string]Artifact{"linux-amd64": {}}}
+ err := Ship(context.Background(), s, m, map[string][]byte{"linux-amd64": []byte("different")})
+ if err == nil {
+ t.Fatalf("Ship accepted different content under an existing version")
+ }
+}
+
+func TestPromoteRequiresShippedVersion(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ priv, _ := testKeys(t)
+ err := Promote(context.Background(), s, priv, "myapp", "stable", "ghost1")
+ if err == nil || !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("Promote of unshipped version: err = %v, want os.ErrNotExist-wrapped", err)
+ }
+}
+
+func TestTamperedPointerRejected(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ priv, pub := testKeys(t)
+ shipRelease(t, s, "myapp", "v1abc", "linux-amd64", []byte("bin"))
+ if err := Promote(ctx, s, priv, "myapp", "stable", "v1abc"); err != nil {
+ t.Fatalf("Promote: %v", err)
+ }
+
+ key := ChannelKey("myapp", "stable")
+ raw, etag, err := s.Get(ctx, key)
+ if err != nil {
+ t.Fatalf("Get pointer: %v", err)
+ }
+ var sw signedWrapper
+ if err := json.Unmarshal(raw, &sw); err != nil {
+ t.Fatalf("unmarshal wrapper: %v", err)
+ }
+
+ // Flip a byte in the signature.
+ sig := []byte(sw.Sig)
+ if sig[10] == 'A' {
+ sig[10] = 'B'
+ } else {
+ sig[10] = 'A'
+ }
+ tampered, _ := json.Marshal(signedWrapper{Payload: sw.Payload, Sig: string(sig)})
+ if _, err := s.Put(ctx, key, tampered, etag); err != nil {
+ t.Fatalf("plant tampered pointer: %v", err)
+ }
+ if _, err := Channel(ctx, s, pub, "myapp", "stable"); err == nil {
+ t.Fatalf("Channel accepted a pointer with a flipped signature byte")
+ }
+
+ // Flip a byte in the payload instead (signature intact) — the signature
+ // covers the exact base64 payload bytes, so this must fail too.
+ payload := []byte(sw.Payload)
+ if payload[10] == 'A' {
+ payload[10] = 'B'
+ } else {
+ payload[10] = 'A'
+ }
+ raw2, etag2, _ := s.Get(ctx, key)
+ _ = raw2
+ tampered2, _ := json.Marshal(signedWrapper{Payload: string(payload), Sig: sw.Sig})
+ if _, err := s.Put(ctx, key, tampered2, etag2); err != nil {
+ t.Fatalf("plant tampered payload: %v", err)
+ }
+ if _, err := Channel(ctx, s, pub, "myapp", "stable"); err == nil {
+ t.Fatalf("Channel accepted a pointer with a flipped payload byte")
+ }
+
+ // And a validly-signed pointer copied from ANOTHER channel's key is a
+ // cross-channel replay — rejected because app/channel live under the
+ // signature.
+ if err := Promote(ctx, s, priv, "myapp", "canary/evil", "v1abc"); err != nil {
+ t.Fatalf("Promote canary: %v", err)
+ }
+ canaryRaw, _, _ := s.Get(ctx, ChannelKey("myapp", "canary/evil"))
+ _, curEtag, _ := s.Get(ctx, key)
+ if _, err := s.Put(ctx, key, canaryRaw, curEtag); err != nil {
+ t.Fatalf("plant replayed pointer: %v", err)
+ }
+ if _, err := Channel(ctx, s, pub, "myapp", "stable"); err == nil {
+ t.Fatalf("Channel accepted a pointer replayed from another channel")
+ }
+}
+
+func TestPromoteHistoryCapTen(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ priv, pub := testKeys(t)
+ for i := 1; i <= 12; i++ {
+ v := fmt.Sprintf("v%02d", i)
+ shipRelease(t, s, "myapp", v, "linux-amd64", []byte("bin "+v))
+ if err := Promote(ctx, s, priv, "myapp", "stable", v); err != nil {
+ t.Fatalf("Promote %s: %v", v, err)
+ }
+ }
+ ptr, err := Channel(ctx, s, pub, "myapp", "stable")
+ if err != nil {
+ t.Fatalf("Channel: %v", err)
+ }
+ if ptr.Version != "v12" {
+ t.Fatalf("Version = %s, want v12", ptr.Version)
+ }
+ if len(ptr.History) != 10 {
+ t.Fatalf("History has %d entries, want 10: %v", len(ptr.History), ptr.History)
+ }
+ if ptr.History[0] != "v11" || ptr.History[9] != "v02" {
+ t.Fatalf("History = %v, want newest-first v11..v02", ptr.History)
+ }
+}
+
+func TestRollbackSwapsAndReturns(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ priv, pub := testKeys(t)
+ shipRelease(t, s, "myapp", "v1", "linux-amd64", []byte("one"))
+ shipRelease(t, s, "myapp", "v2", "linux-amd64", []byte("two"))
+ if err := Promote(ctx, s, priv, "myapp", "stable", "v1"); err != nil {
+ t.Fatalf("Promote v1: %v", err)
+ }
+
+ // Nothing to roll back to yet.
+ if err := Rollback(ctx, s, priv, "myapp", "stable"); err == nil {
+ t.Fatalf("Rollback with empty history succeeded")
+ }
+
+ if err := Promote(ctx, s, priv, "myapp", "stable", "v2"); err != nil {
+ t.Fatalf("Promote v2: %v", err)
+ }
+ if err := Rollback(ctx, s, priv, "myapp", "stable"); err != nil {
+ t.Fatalf("Rollback: %v", err)
+ }
+ ptr, err := Channel(ctx, s, pub, "myapp", "stable")
+ if err != nil {
+ t.Fatalf("Channel: %v", err)
+ }
+ if ptr.Version != "v1" || len(ptr.History) != 1 || ptr.History[0] != "v2" {
+ t.Fatalf("after rollback: %+v, want v1 with history [v2]", ptr)
+ }
+ // A second rollback returns to where you were (the swap semantics).
+ if err := Rollback(ctx, s, priv, "myapp", "stable"); err != nil {
+ t.Fatalf("second Rollback: %v", err)
+ }
+ ptr, _ = Channel(ctx, s, pub, "myapp", "stable")
+ if ptr.Version != "v2" || len(ptr.History) != 1 || ptr.History[0] != "v1" {
+ t.Fatalf("after second rollback: %+v, want v2 with history [v1]", ptr)
+ }
+}
+
+func TestChannelWrongKeyRejected(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ priv, _ := testKeys(t)
+ _, otherPub := testKeys(t)
+ shipRelease(t, s, "myapp", "v1", "linux-amd64", []byte("one"))
+ if err := Promote(ctx, s, priv, "myapp", "stable", "v1"); err != nil {
+ t.Fatalf("Promote: %v", err)
+ }
+ if _, err := Channel(ctx, s, otherPub, "myapp", "stable"); err == nil {
+ t.Fatalf("Channel verified against the wrong public key")
+ }
+}
+
+func TestValidChannel(t *testing.T) {
+ for c, want := range map[string]bool{
+ "stable": true, "beta": true, "canary/feature-x": true,
+ "canary/..": false, "a/b/c": false, "": false, "sta ble": false,
+ } {
+ if got := ValidChannel(c); got != want {
+ t.Errorf("ValidChannel(%q) = %v, want %v", c, got, want)
+ }
+ }
+}
+
+// Guard against the PromotedAt stamp being mistaken for a trust input
+// anywhere: the bake clock must run off the box's own sightings, and the
+// stamp is informational. This just pins that signPointer stamps it.
+func TestSignStampsPromotedAt(t *testing.T) {
+ priv, pub := testKeys(t)
+ p := &ChannelPointer{App: "myapp", Channel: "stable", Version: "v1"}
+ before := time.Now().Add(-time.Second)
+ raw, err := signPointer(priv, p)
+ if err != nil {
+ t.Fatalf("signPointer: %v", err)
+ }
+ got, err := verifyPointer(pub, raw, "myapp", "stable")
+ if err != nil {
+ t.Fatalf("verifyPointer: %v", err)
+ }
+ if got.PromotedAt.Before(before) {
+ t.Fatalf("PromotedAt = %v, want stamped at signing time", got.PromotedAt)
+ }
+}
diff --git a/internal/release/store.go b/internal/release/store.go
new file mode 100644
index 000000000..c2abaf452
--- /dev/null
+++ b/internal/release/store.go
@@ -0,0 +1,280 @@
+// Store — the deployment bucket behind the release plane: S3 in production,
+// a plain directory in tests. CAS discipline (If-Match / If-None-Match)
+// ported from Eleven internal/idalloc/idalloc.go (the litestream leaser's
+// conditional-write pattern); not-found folding from Eleven s3blobstore.go.
+package release
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "net/http"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+ "sync"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awsconfig "github.com/aws/aws-sdk-go-v2/config"
+ awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/smithy-go"
+)
+
+// ErrCASConflict is a conditional Put that lost: the object already existed
+// (ifMatch "") or moved on since the etag was read (ifMatch stale). Callers
+// re-read and retry; they never overwrite blind.
+var ErrCASConflict = errors.New("store: compare-and-swap conflict")
+
+// Store is the deployment bucket. Get's not-found is os.ErrNotExist-wrapped
+// so callers use errors.Is; Put with ifMatch "" is create-only
+// (If-None-Match: *), non-empty is replace-only (If-Match: <etag>), and a
+// lost race is ErrCASConflict either way.
+type Store interface {
+ Get(ctx context.Context, key string) ([]byte, string, error) // body, etag
+ Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) // new etag
+ List(ctx context.Context, prefix string) ([]string, error)
+}
+
+// ---- DirStore ---------------------------------------------------------------
+
+// DirStore keeps objects as files under a root directory — the test double
+// and the single-box no-AWS deployment shape. The etag is the hex sha256 of
+// the content (content-derived, so it survives process restarts with no
+// sidecar files). CAS is enforced under one process-wide mutex plus
+// temp-file + rename writes: single-process fidelity, which is what the
+// tests and a one-box deployment need.
+type DirStore struct {
+ root string
+ mu sync.Mutex
+}
+
+func NewDirStore(root string) *DirStore { return &DirStore{root: root} }
+
+// path maps a bucket key onto the root dir, refusing traversal — a key is
+// bucket data, never a filesystem instruction.
+func (d *DirStore) path(key string) (string, error) {
+ if key == "" || strings.HasPrefix(key, "/") {
+ return "", fmt.Errorf("dirstore: invalid key %q", key)
+ }
+ for _, seg := range strings.Split(key, "/") {
+ if seg == "" || seg == "." || seg == ".." {
+ return "", fmt.Errorf("dirstore: invalid key %q", key)
+ }
+ }
+ return filepath.Join(d.root, filepath.FromSlash(key)), nil
+}
+
+func etagOf(body []byte) string {
+ sum := sha256.Sum256(body)
+ return hex.EncodeToString(sum[:])
+}
+
+func (d *DirStore) Get(_ context.Context, key string) ([]byte, string, error) {
+ p, err := d.path(key)
+ if err != nil {
+ return nil, "", err
+ }
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ b, err := os.ReadFile(p)
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ return nil, "", fmt.Errorf("dirstore: %s: %w", key, os.ErrNotExist)
+ }
+ return nil, "", err
+ }
+ return b, etagOf(b), nil
+}
+
+func (d *DirStore) Put(_ context.Context, key string, body []byte, ifMatch string) (string, error) {
+ p, err := d.path(key)
+ if err != nil {
+ return "", err
+ }
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ cur, err := os.ReadFile(p)
+ exists := err == nil
+ if err != nil && !errors.Is(err, fs.ErrNotExist) {
+ return "", err
+ }
+ // The CAS gate, mirroring S3's semantics exactly: "" = create-only,
+ // non-empty = the object must still carry that etag.
+ if ifMatch == "" && exists {
+ return "", fmt.Errorf("dirstore: %s exists: %w", key, ErrCASConflict)
+ }
+ if ifMatch != "" && (!exists || etagOf(cur) != ifMatch) {
+ return "", fmt.Errorf("dirstore: %s moved on: %w", key, ErrCASConflict)
+ }
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ return "", err
+ }
+ // Temp + rename so a crashed write never leaves a half-object a reader
+ // could mistake for the real thing.
+ tmp := p + ".tmp"
+ if err := os.WriteFile(tmp, body, 0o644); err != nil {
+ return "", err
+ }
+ if err := os.Rename(tmp, p); err != nil {
+ os.Remove(tmp)
+ return "", err
+ }
+ return etagOf(body), nil
+}
+
+func (d *DirStore) List(_ context.Context, prefix string) ([]string, error) {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ var out []string
+ err := filepath.WalkDir(d.root, func(p string, de fs.DirEntry, err error) error {
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ return nil // an empty store lists empty, like an empty bucket
+ }
+ return err
+ }
+ if de.IsDir() || strings.HasSuffix(p, ".tmp") {
+ return nil
+ }
+ rel, err := filepath.Rel(d.root, p)
+ if err != nil {
+ return err
+ }
+ key := filepath.ToSlash(rel)
+ if strings.HasPrefix(key, prefix) {
+ out = append(out, key)
+ }
+ return nil
+ })
+ sort.Strings(out)
+ return out, err
+}
+
+// ---- S3Store ----------------------------------------------------------------
+
+// S3Store is the production bucket via aws-sdk-go-v2, with the conditional
+// headers doing the CAS work server-side. Compiles always; exercised live
+// only when CARLOS_TEST_S3_BUCKET is set (see release_test.go).
+type S3Store struct {
+ Client *awss3.Client
+ Bucket string
+}
+
+// NewS3Store builds a store on the standard AWS credential/region chain.
+// CARLOS_S3_ENDPOINT (optional) points at an S3-compatible (R2, MinIO) and
+// implies path-style addressing; checksums go when-required because
+// default-on flexible checksums wrap bodies in aws-chunked framing some
+// compatibles reject (Eleven s3blobstore.go's R2 lesson).
+func NewS3Store(ctx context.Context, bucket string) (*S3Store, error) {
+ cfg, err := awsconfig.LoadDefaultConfig(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("s3store: load aws config: %w", err)
+ }
+ client := awss3.NewFromConfig(cfg, func(o *awss3.Options) {
+ if ep := strings.TrimSpace(os.Getenv("CARLOS_S3_ENDPOINT")); ep != "" {
+ o.BaseEndpoint = &ep
+ o.UsePathStyle = true
+ }
+ o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+ o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
+ })
+ return &S3Store{Client: client, Bucket: bucket}, nil
+}
+
+// s3NotFound folds the shapes a missing object answers with (NoSuchKey on
+// GET, NotFound on HEAD, bare 404s from S3-compatibles) into one test.
+func s3NotFound(err error) bool {
+ var apiErr smithy.APIError
+ if errors.As(err, &apiErr) {
+ switch apiErr.ErrorCode() {
+ case "NoSuchKey", "NotFound", "404":
+ return true
+ }
+ }
+ return false
+}
+
+// s3CASConflict reports whether an error is a conditional-write loss: 412
+// (If-Match / If-None-Match precondition failed) or 409 (S3's
+// ConditionalRequestConflict when two conditional writes race). Matched by
+// both the S3 error code and the raw HTTP status, so an S3-compatible that
+// only sets one of them still classifies correctly.
+func s3CASConflict(err error) bool {
+ var apiErr smithy.APIError
+ if errors.As(err, &apiErr) {
+ switch apiErr.ErrorCode() {
+ case "PreconditionFailed", "ConditionalRequestConflict":
+ return true
+ }
+ }
+ var httpErr interface{ HTTPStatusCode() int }
+ if errors.As(err, &httpErr) {
+ switch httpErr.HTTPStatusCode() {
+ case http.StatusPreconditionFailed, http.StatusConflict:
+ return true
+ }
+ }
+ return false
+}
+
+func (s *S3Store) Get(ctx context.Context, key string) ([]byte, string, error) {
+ out, err := s.Client.GetObject(ctx, &awss3.GetObjectInput{Bucket: &s.Bucket, Key: &key})
+ if err != nil {
+ if s3NotFound(err) {
+ return nil, "", fmt.Errorf("s3store: %s: %w", key, os.ErrNotExist)
+ }
+ return nil, "", err
+ }
+ defer out.Body.Close()
+ b, err := io.ReadAll(out.Body)
+ if err != nil {
+ return nil, "", err
+ }
+ return b, aws.ToString(out.ETag), nil
+}
+
+func (s *S3Store) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
+ in := &awss3.PutObjectInput{
+ Bucket: &s.Bucket,
+ Key: &key,
+ Body: bytes.NewReader(body),
+ ContentLength: aws.Int64(int64(len(body))),
+ }
+ if ifMatch == "" {
+ in.IfNoneMatch = aws.String("*") // create-only: 412 if someone created it first
+ } else {
+ in.IfMatch = aws.String(ifMatch) // replace-only: 412 if the object moved on
+ }
+ out, err := s.Client.PutObject(ctx, in)
+ if err != nil {
+ if s3CASConflict(err) {
+ return "", fmt.Errorf("s3store: %s: %w", key, ErrCASConflict)
+ }
+ return "", err
+ }
+ return aws.ToString(out.ETag), nil
+}
+
+func (s *S3Store) List(ctx context.Context, prefix string) ([]string, error) {
+ var out []string
+ p := awss3.NewListObjectsV2Paginator(s.Client, &awss3.ListObjectsV2Input{
+ Bucket: &s.Bucket,
+ Prefix: &prefix,
+ })
+ for p.HasMorePages() {
+ page, err := p.NextPage(ctx)
+ if err != nil {
+ return nil, err
+ }
+ for _, o := range page.Contents {
+ out = append(out, aws.ToString(o.Key))
+ }
+ }
+ return out, nil
+}
diff --git a/internal/release/store_test.go b/internal/release/store_test.go
new file mode 100644
index 000000000..50e456ab3
--- /dev/null
+++ b/internal/release/store_test.go
@@ -0,0 +1,137 @@
+package release
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "testing"
+ "time"
+)
+
+func TestPutIfNoneMatchConflict(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ if _, err := s.Put(ctx, "a/b.json", []byte("one"), ""); err != nil {
+ t.Fatalf("create: %v", err)
+ }
+ // A second create-only Put of an existing key must lose.
+ if _, err := s.Put(ctx, "a/b.json", []byte("two"), ""); !errors.Is(err, ErrCASConflict) {
+ t.Fatalf("create over existing: err = %v, want ErrCASConflict", err)
+ }
+ // The loser changed nothing.
+ body, _, err := s.Get(ctx, "a/b.json")
+ if err != nil || string(body) != "one" {
+ t.Fatalf("Get = %q, %v; want %q", body, err, "one")
+ }
+}
+
+func TestPutIfMatchStaleEtagFails(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ etag1, err := s.Put(ctx, "k", []byte("one"), "")
+ if err != nil {
+ t.Fatalf("create: %v", err)
+ }
+ etag2, err := s.Put(ctx, "k", []byte("two"), etag1)
+ if err != nil {
+ t.Fatalf("CAS replace: %v", err)
+ }
+ if etag2 == etag1 {
+ t.Fatalf("etag didn't change on replace")
+ }
+ // Replaying the old etag must lose.
+ if _, err := s.Put(ctx, "k", []byte("three"), etag1); !errors.Is(err, ErrCASConflict) {
+ t.Fatalf("stale etag: err = %v, want ErrCASConflict", err)
+ }
+ // And a CAS write against a key that doesn't exist must lose too.
+ if _, err := s.Put(ctx, "missing", []byte("x"), etag1); !errors.Is(err, ErrCASConflict) {
+ t.Fatalf("CAS on missing key: err = %v, want ErrCASConflict", err)
+ }
+ body, _, _ := s.Get(ctx, "k")
+ if string(body) != "two" {
+ t.Fatalf("Get = %q, want %q", body, "two")
+ }
+}
+
+func TestGetNotFoundWrapsErrNotExist(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ if _, _, err := s.Get(context.Background(), "no/such/key"); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("err = %v, want os.ErrNotExist-wrapped", err)
+ }
+}
+
+func TestListPrefix(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ for _, k := range []string{"apps/x/releases/v1/manifest.json", "apps/x/releases/v1/bin", "apps/x/channels/stable.json", "apps/y/one"} {
+ if _, err := s.Put(ctx, k, []byte(k), ""); err != nil {
+ t.Fatalf("put %s: %v", k, err)
+ }
+ }
+ keys, err := s.List(ctx, "apps/x/releases/v1/")
+ if err != nil {
+ t.Fatalf("list: %v", err)
+ }
+ want := []string{"apps/x/releases/v1/bin", "apps/x/releases/v1/manifest.json"}
+ if len(keys) != len(want) || keys[0] != want[0] || keys[1] != want[1] {
+ t.Fatalf("List = %v, want %v", keys, want)
+ }
+ // An empty store (or unmatched prefix) lists empty, like an empty bucket.
+ keys, err = s.List(ctx, "nothing/")
+ if err != nil || len(keys) != 0 {
+ t.Fatalf("List(nothing/) = %v, %v; want empty", keys, err)
+ }
+}
+
+func TestDirStoreRejectsTraversal(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ for _, k := range []string{"../evil", "a/../../evil", "/abs", "a//b", ""} {
+ if _, err := s.Put(ctx, k, []byte("x"), ""); err == nil {
+ t.Fatalf("Put(%q) accepted a traversal key", k)
+ }
+ }
+}
+
+// TestS3StoreLive exercises the production store against a real bucket —
+// only when CARLOS_TEST_S3_BUCKET is set; the type otherwise just compiles.
+func TestS3StoreLive(t *testing.T) {
+ bucket := os.Getenv("CARLOS_TEST_S3_BUCKET")
+ if bucket == "" {
+ t.Skip("CARLOS_TEST_S3_BUCKET not set — skipping live S3Store test")
+ }
+ ctx := context.Background()
+ s, err := NewS3Store(ctx, bucket)
+ if err != nil {
+ t.Fatalf("NewS3Store: %v", err)
+ }
+ key := fmt.Sprintf("carlos-release-test/%d/ptr.json", time.Now().UnixNano())
+
+ if _, _, err := s.Get(ctx, key); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("Get missing: err = %v, want os.ErrNotExist-wrapped", err)
+ }
+ etag, err := s.Put(ctx, key, []byte("one"), "")
+ if err != nil {
+ t.Fatalf("create: %v", err)
+ }
+ if _, err := s.Put(ctx, key, []byte("two"), ""); !errors.Is(err, ErrCASConflict) {
+ t.Fatalf("create over existing: err = %v, want ErrCASConflict", err)
+ }
+ body, gotEtag, err := s.Get(ctx, key)
+ if err != nil || string(body) != "one" || gotEtag != etag {
+ t.Fatalf("Get = %q, %q, %v; want %q, %q", body, gotEtag, err, "one", etag)
+ }
+ etag2, err := s.Put(ctx, key, []byte("two"), etag)
+ if err != nil {
+ t.Fatalf("CAS replace: %v", err)
+ }
+ if _, err := s.Put(ctx, key, []byte("three"), etag); !errors.Is(err, ErrCASConflict) {
+ t.Fatalf("stale etag: err = %v, want ErrCASConflict", err)
+ }
+ _ = etag2
+ keys, err := s.List(ctx, key)
+ if err != nil || len(keys) != 1 || keys[0] != key {
+ t.Fatalf("List = %v, %v; want [%s]", keys, err, key)
+ }
+}
diff --git a/internal/release/versions.go b/internal/release/versions.go
new file mode 100644
index 000000000..49a872610
--- /dev/null
+++ b/internal/release/versions.go
@@ -0,0 +1,236 @@
+// The on-box version store — ported from Eleven versions.go.
+//
+// Versioned, side-by-side deploys: the box runs one binary per host, but
+// they need not be the SAME binary. Each host's unit execs
+// <opt>/live/<host>, a symlink into a content store at
+// <opt>/versions/<ver>/<app>. Swapping one link (and restarting one unit)
+// moves one host to a new build while every other host stays put — so a new
+// build can be canaried on one host before it's promoted fleet-wide, and
+// rolled back per host.
+//
+// The opt dir defaults from CARLOS_OPT_DIR (default /opt/carlos), but every
+// function takes the base dir as a parameter so tests never touch /opt.
+package release
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "os"
+ "path/filepath"
+ "regexp"
+ "time"
+)
+
+// OptDir is the on-box install root: CARLOS_OPT_DIR, default /opt/carlos.
+// Resolved at call time (not init) so a test's Setenv is honoured.
+func OptDir() string {
+ if d := os.Getenv("CARLOS_OPT_DIR"); d != "" {
+ return d
+ }
+ return "/opt/carlos"
+}
+
+func versionsDir(optDir string) string { return filepath.Join(optDir, "versions") }
+func liveDir(optDir string) string { return filepath.Join(optDir, "live") }
+
+// VersionPath is where a staged build of app lives: <opt>/versions/<ver>/<app>.
+func VersionPath(optDir, ver, app string) string {
+ return filepath.Join(versionsDir(optDir), ver, app)
+}
+
+// LiveLink is the per-host pointer the unit execs (<opt>/live/<host>) — a
+// symlink to a VersionPath. Swapping it atomically is how one host moves
+// between versions independently of the others.
+func LiveLink(optDir, host string) string { return filepath.Join(liveDir(optDir), host) }
+
+// versionRe keeps a version usable as a directory name (no slashes, no
+// traversal).
+var versionRe = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`)
+
+func ValidVersion(v string) bool {
+ return v != "" && len(v) <= 64 && v != "." && v != ".." && versionRe.MatchString(v)
+}
+
+// CopyFileAtomic copies src to dst via a temp file + rename, so a reader
+// never sees a half-written binary (and a crashed copy leaves the old one
+// intact).
+func CopyFileAtomic(src, dst string, mode os.FileMode) error {
+ in, err := os.Open(src)
+ if err != nil {
+ return err
+ }
+ defer in.Close()
+ tmp := dst + ".tmp"
+ out, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
+ if err != nil {
+ return err
+ }
+ if _, err := io.Copy(out, in); err != nil {
+ out.Close()
+ os.Remove(tmp)
+ return err
+ }
+ if err := out.Close(); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ // O_CREATE's mode is masked by umask; set it explicitly so the binary is +x.
+ if err := os.Chmod(tmp, mode); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ if err := os.Rename(tmp, dst); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ return nil
+}
+
+// WriteFileAtomic is CopyFileAtomic for in-memory bytes (a downloaded,
+// sha256-verified artifact).
+func WriteFileAtomic(dst string, data []byte, mode os.FileMode) error {
+ tmp := dst + ".tmp"
+ if err := os.WriteFile(tmp, data, mode); err != nil {
+ return err
+ }
+ if err := os.Chmod(tmp, mode); err != nil { // umask, as above
+ os.Remove(tmp)
+ return err
+ }
+ if err := os.Rename(tmp, dst); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ return nil
+}
+
+// StageArtifact puts verified artifact bytes into the store under ver, so
+// hosts can be pointed at it. Idempotent: re-staging overwrites.
+func StageArtifact(optDir, ver, app string, data []byte) error {
+ if !ValidVersion(ver) {
+ return fmt.Errorf("invalid version %q (allowed: letters, digits, . _ -)", ver)
+ }
+ if !ValidApp(app) {
+ return fmt.Errorf("invalid app %q", app)
+ }
+ dir := filepath.Join(versionsDir(optDir), ver)
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ return err
+ }
+ // MkdirAll's mode is masked by the caller's umask, so a tight operator
+ // umask (e.g. 077 from a hardened shell) leaves the version dir 0700 —
+ // the sandboxed service user then can't traverse in to exec its binary
+ // (203/EXEC "Permission denied", found staging a canary on Eleven).
+ // Force it, like the write below forces the binary's +x.
+ if err := os.Chmod(dir, 0o755); err != nil {
+ return err
+ }
+ return WriteFileAtomic(VersionPath(optDir, ver, app), data, 0o755)
+}
+
+// PointLive atomically repoints a host's live symlink at ver's build of app.
+// It verifies ver is staged first, so a typo can't leave a dangling link
+// that would fail the unit's ExecStart on the next restart.
+func PointLive(optDir, host, ver, app string) error {
+ target := VersionPath(optDir, ver, app)
+ if _, err := os.Stat(target); err != nil {
+ return fmt.Errorf("version %q of %s is not staged (%s)", ver, app, target)
+ }
+ if err := os.MkdirAll(liveDir(optDir), 0o755); err != nil {
+ return err
+ }
+ link := LiveLink(optDir, host)
+ tmp := link + ".tmp"
+ _ = os.Remove(tmp) // a leftover temp link from a crashed swap
+ if err := os.Symlink(target, tmp); err != nil {
+ return err
+ }
+ if err := os.Rename(tmp, link); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ return nil
+}
+
+// LiveVersion reports the version a host's live symlink currently points at,
+// or "" (no error) if the host hasn't been put under version management yet.
+func LiveVersion(optDir, host string) (string, error) {
+ target, err := os.Readlink(LiveLink(optDir, host))
+ if err != nil {
+ if os.IsNotExist(err) {
+ return "", nil
+ }
+ return "", err
+ }
+ // target = <opt>/versions/<ver>/<app> -> the parent dir's name is <ver>.
+ return filepath.Base(filepath.Dir(target)), nil
+}
+
+// WaitHealthyVersion waits until the host behind socket not only answers but
+// reports the expected build at /api/version — so a deploy fails loudly if
+// the process is still the old binary (a stale socket, or a restart that
+// didn't take), rather than silently "succeeding".
+func WaitHealthyVersion(socket, wantBuild string, timeout time.Duration) error {
+ deadline := time.Now().Add(timeout)
+ client := unixHTTPClient(socket)
+ var last error
+ for time.Now().Before(deadline) {
+ resp, err := client.Get("http://carlos/api/version")
+ if err == nil {
+ var body struct {
+ Build string `json:"build"`
+ }
+ derr := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&body)
+ resp.Body.Close()
+ if derr == nil && body.Build == wantBuild {
+ return nil
+ }
+ if derr != nil {
+ last = fmt.Errorf("read /api/version: %v", derr)
+ } else {
+ last = fmt.Errorf("live build %q != expected %q", body.Build, wantBuild)
+ }
+ } else {
+ last = err
+ }
+ time.Sleep(150 * time.Millisecond)
+ }
+ return fmt.Errorf("host didn't report build %q within %s: %v", wantBuild, timeout, last)
+}
+
+// ProbeVersion reads the build the process behind a socket is ACTUALLY
+// serving (its /api/version), not the symlink target — how a status probe
+// catches an awake hibernating host that repointed its symlink but still
+// runs the old binary. "" + ok=false when the socket doesn't answer (asleep,
+// or no such host).
+func ProbeVersion(socket string) (build string, ok bool) {
+ resp, err := unixHTTPClient(socket).Get("http://carlos/api/version")
+ if err != nil {
+ return "", false
+ }
+ defer resp.Body.Close()
+ var body struct {
+ Build string `json:"build"`
+ }
+ if json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&body) != nil {
+ return "", false
+ }
+ return body.Build, body.Build != ""
+}
+
+// unixHTTPClient dials a unix socket over HTTP — the edge<->host transport,
+// reused by the health checks.
+func unixHTTPClient(socket string) *http.Client {
+ return &http.Client{
+ Timeout: 3 * time.Second,
+ Transport: &http.Transport{
+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+ return (&net.Dialer{}).DialContext(ctx, "unix", socket)
+ },
+ },
+ }
+}
diff --git a/internal/release/versions_test.go b/internal/release/versions_test.go
new file mode 100644
index 000000000..73f805466
--- /dev/null
+++ b/internal/release/versions_test.go
@@ -0,0 +1,112 @@
+package release
+
+import (
+ "errors"
+ "net"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestStagePointLiveRoundTrip(t *testing.T) {
+ opt := t.TempDir()
+ if err := StageArtifact(opt, "v1abc", "myapp", []byte("the binary")); err != nil {
+ t.Fatalf("StageArtifact: %v", err)
+ }
+ p := VersionPath(opt, "v1abc", "myapp")
+ st, err := os.Stat(p)
+ if err != nil {
+ t.Fatalf("staged file: %v", err)
+ }
+ if st.Mode().Perm() != 0o755 {
+ t.Fatalf("staged mode = %v, want 0755 (+x survives umask)", st.Mode().Perm())
+ }
+ // An unmanaged host reads as "" with no error.
+ if v, err := LiveVersion(opt, "h.example.com"); err != nil || v != "" {
+ t.Fatalf("LiveVersion before pointing = %q, %v", v, err)
+ }
+ if err := PointLive(opt, "h.example.com", "v1abc", "myapp"); err != nil {
+ t.Fatalf("PointLive: %v", err)
+ }
+ v, err := LiveVersion(opt, "h.example.com")
+ if err != nil || v != "v1abc" {
+ t.Fatalf("LiveVersion = %q, %v; want v1abc", v, err)
+ }
+ // Repointing at an unstaged version is refused (no dangling links).
+ if err := PointLive(opt, "h.example.com", "vGhost", "myapp"); err == nil {
+ t.Fatalf("PointLive accepted an unstaged version")
+ }
+ // The refusal left the link untouched.
+ if v, _ := LiveVersion(opt, "h.example.com"); v != "v1abc" {
+ t.Fatalf("live link moved to %q after refused repoint", v)
+ }
+ // A second staged version swaps atomically.
+ if err := StageArtifact(opt, "v2def", "myapp", []byte("newer binary")); err != nil {
+ t.Fatalf("StageArtifact v2: %v", err)
+ }
+ if err := PointLive(opt, "h.example.com", "v2def", "myapp"); err != nil {
+ t.Fatalf("PointLive v2: %v", err)
+ }
+ if v, _ := LiveVersion(opt, "h.example.com"); v != "v2def" {
+ t.Fatalf("LiveVersion = %q, want v2def", v)
+ }
+}
+
+func TestValidVersion(t *testing.T) {
+ for v, want := range map[string]bool{
+ "abc1234": true, "1.2.3": true, "a_b-c.d": true,
+ "": false, ".": false, "..": false, "a/b": false, "a b": false,
+ } {
+ if got := ValidVersion(v); got != want {
+ t.Errorf("ValidVersion(%q) = %v, want %v", v, got, want)
+ }
+ }
+}
+
+func TestOptDirEnvOverride(t *testing.T) {
+ t.Setenv("CARLOS_OPT_DIR", "/x/y")
+ if d := OptDir(); d != "/x/y" {
+ t.Fatalf("OptDir = %q, want /x/y", d)
+ }
+ t.Setenv("CARLOS_OPT_DIR", "")
+ if d := OptDir(); d != "/opt/carlos" {
+ t.Fatalf("OptDir = %q, want /opt/carlos", d)
+ }
+}
+
+func TestProbeAndWaitHealthyVersion(t *testing.T) {
+ sock := filepath.Join(t.TempDir(), "h.sock")
+ ln, err := net.Listen("unix", sock)
+ if err != nil {
+ if errors.Is(err, os.ErrPermission) || strings.Contains(err.Error(), "operation not permitted") {
+ t.Skipf("unix sockets unavailable in this sandbox: %v", err)
+ }
+ t.Fatalf("listen: %v", err)
+ }
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/version", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`{"build":"v1abc"}`))
+ })
+ srv := &http.Server{Handler: mux}
+ go srv.Serve(ln)
+ defer srv.Close()
+
+ build, ok := ProbeVersion(sock)
+ if !ok || build != "v1abc" {
+ t.Fatalf("ProbeVersion = %q, %v; want v1abc, true", build, ok)
+ }
+ if err := WaitHealthyVersion(sock, "v1abc", 3*time.Second); err != nil {
+ t.Fatalf("WaitHealthyVersion: %v", err)
+ }
+ // A wrong expected build times out with the mismatch surfaced.
+ if err := WaitHealthyVersion(sock, "vOther", 400*time.Millisecond); err == nil {
+ t.Fatalf("WaitHealthyVersion accepted the wrong build")
+ }
+ // A dead socket probes false.
+ if _, ok := ProbeVersion(filepath.Join(t.TempDir(), "dead.sock")); ok {
+ t.Fatalf("ProbeVersion answered on a dead socket")
+ }
+}