amadan

carlos / platform Public

Merge pull request #119 from carlosframework/restart-verb

carlos restart: cycle an app's processes with no box access (#113)
Paul Campbell de23f6ae197761677170f11b8fafaacdac298655 parent c8b2eedparent 7f11c74
22 files changed, +2441 −33
  • apimode.go +39 −0
  • internal/console/apicli.go +86 −0
  • internal/console/apirestart_test.go +177 −0
  • internal/console/authcli.go +5 −0
  • internal/host/agent.go +81 −1
  • internal/host/bootstrap.go +6 −4
  • internal/host/host_test.go +2 −0
  • internal/host/probes.go +19 −0
  • internal/host/restart.go +650 −0
  • internal/host/restart_test.go +706 −0
  • internal/host/units/carlos-restart.path +27 −0
  • internal/host/units/carlos-restart.service +53 −0
  • internal/release/restartgen.go +107 −0
  • internal/release/restartgen_test.go +135 −0
  • internal/status/contract_test.go +9 −2
  • internal/status/dryrun.go +10 −8
  • internal/status/fleet.go +14 −8
  • internal/status/snapshot.go +26 −10
  • internal/status/status_test.go +47 −0
  • main.go +3 −0
  • restart.go +86 −0
  • restart_test.go +153 −0
diff --git a/apimode.go b/apimode.go
index d309e3759..ad4daf82e 100644
--- a/apimode.go
+++ b/apimode.go
@@ -414,6 +414,45 @@ func (s *apiSession) rollback(ctx context.Context, a apiAcct, app, channel strin
return apiErrorFromBody(status, body)
}
+// apiRestartResult is what `carlos restart` prints: the console's CANONICAL
+// account sqid and app name, since the segment the caller typed may have been
+// a name resolved server-side.
+type apiRestartResult struct {
+ Account string `json:"account"`
+ App string `json:"app"`
+}
+
+// restart asks the console to request a process cycle for app on every box
+// that serves it (issue #113). Mirrors trashApp's shape — postJSON, a 200
+// decodes, anything else is apiErrorFromBody's job — with ONE addition, the
+// 404.
+//
+// A 404 here is genuinely ambiguous, and pretending otherwise would be a lie
+// in one direction or the other. An old console has no /restart route at all,
+// so net/http's mux answers exactly the same byte-identical "404 page not
+// found" apiAccountApp gives an unknown account, someone else's account, or an
+// app that isn't there — there is nothing in the response to tell them apart.
+// So the message names BOTH, shortest fix first, the same way restoreApp
+// widens its own 404 rather than inventing certainty it doesn't have.
+func (s *apiSession) restart(ctx context.Context, a apiAcct, app string) (apiRestartResult, error) {
+ status, body, err := s.postJSON(ctx, apiPath(a, app, "/restart"), struct{}{})
+ if err != nil {
+ return apiRestartResult{}, err
+ }
+ if status != http.StatusOK {
+ if status == http.StatusNotFound {
+ return apiRestartResult{}, fmt.Errorf("%w — or this console predates `carlos restart` "+
+ "(no restart door on it at all), in which case redeploy the console first", notFoundErr())
+ }
+ return apiRestartResult{}, apiErrorFromBody(status, body)
+ }
+ var res apiRestartResult
+ if err := json.Unmarshal(body, &res); err != nil {
+ return apiRestartResult{}, fmt.Errorf("restart: unreadable response: %w", err)
+ }
+ return res, nil
+}
+
// channels reads one app's channel ladder through the console.
func (s *apiSession) channels(ctx context.Context, a apiAcct, app string) ([]apiChannel, error) {
status, body, err := s.getJSON(ctx, apiPath(a, app, "/channels"))
diff --git a/internal/console/apicli.go b/internal/console/apicli.go
index db48921fd..90b54edb6 100644
--- a/internal/console/apicli.go
+++ b/internal/console/apicli.go
@@ -469,6 +469,92 @@ func (s *Server) handleAPIRollback(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]string{"result": "rolled back"})
}
+// handleAPIRestart answers `carlos restart`: touch the app's restart
+// generation, then bump the serial so every box serving it cycles that app's
+// processes within seconds (issue #113). No version moves, no config is
+// written, no bake window applies — this is the member-facing way to cycle a
+// wedged instance, which until now meant a box login the platform's own design
+// rule forbids ("every carlos command works for a user with zero infra
+// access").
+//
+// The BUMP is the delivery mechanism, not decoration. Boxes do not poll
+// restart generations: the agent's existing 2s serial poll is what makes it
+// look, exactly as with a promote or a config write (internal/host/restart.go).
+// A touch that landed without a bump would still converge — on the hourly root
+// pass, up to an hour later — which is the floor, not the promise this door
+// makes.
+//
+// The doors, in order, and why this order — handleAPITrashApp's exactly:
+//
+// token — cliUser's one 401, before any name is resolved.
+// account+app 404 — apiAccountApp: an unknown account, someone else's
+// account and an app that isn't there are one answer, so
+// probing this door teaches a stranger nothing. It also
+// means a TRASHED app cannot be restarted: the live-apps
+// lookup simply doesn't answer for it.
+// rate limit — the shared `write` budget, keyed on the ADDRESS like
+// promote's: a budget belongs to the person, not the
+// terminal. This door needs it more than most — its whole
+// effect is to make processes stop and start, so an
+// unbudgeted one would be a restart loop anyone with a
+// token could aim at their own app.
+// Releases nil — 503. The request IS a bucket object; a console with no
+// bucket has nowhere to put it and must not answer
+// "requested".
+// touch + bump — the object, then the announcement, in that order (a bump
+// for a request that failed to land would wake the fleet
+// for nothing).
+//
+// Audited, unlike a read: this stops and starts customer processes, which
+// anyone reading audit rows must be able to see.
+//
+// No body is read. There is nothing to say beyond the app already named in the
+// path — there is no restart "options" — and a bearer request carries no
+// cookie for a cross-site form to forge.
+//
+// No per-host scoping. The generation is per APP, so every box serving it
+// cycles its own routes for that app; a -host flag would need a per-host
+// object and a way to say what happens to the hosts it doesn't name, and
+// "restart my app" is the request people actually have. Boxes skip their
+// hibernating routes on their own (internal/host/restart.go) — nothing here
+// wakes anything.
+func (s *Server) handleAPIRestart(w http.ResponseWriter, r *http.Request) {
+ addr, acct, app, ok := s.apiAccountApp(w, r)
+ if !ok {
+ return
+ }
+ if !s.cli.write.allow(addr, time.Now()) {
+ jsonError(w, http.StatusTooManyRequests, "too many requests — wait a minute and try again")
+ return
+ }
+ if s.cfg.Releases == nil {
+ jsonError(w, http.StatusServiceUnavailable,
+ "no deployment bucket on this host — this console cannot request a restart")
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
+ defer cancel()
+ if err := release.TouchRestartGen(ctx, s.cfg.Releases, acct.Sqid, app); err != nil {
+ // Logged, not returned: a store error carries bucket names, endpoints
+ // and request ids. The caller gets the fixed sentence every other
+ // bucket failure here gets, and a retry is safe — the touch is
+ // idempotent in the only sense that matters (one more request, one
+ // more cycle).
+ log.Printf("console: api restart %s/%s: %v", acct.Sqid, app, err)
+ jsonError(w, http.StatusBadGateway, "could not request the restart — nothing was cycled; try again")
+ return
+ }
+ s.audit(acct, app, addr, "restart", "requested a restart (cli)")
+ // The request is in the bucket; this is what makes boxes LOOK at it — they
+ // read restart generations only when the serial moves. The difference
+ // between "seconds" and "up to an hour" for the one verb people type when
+ // something is stuck.
+ s.bumpSerial(ctx)
+ writeJSON(w, http.StatusOK, map[string]any{
+ "requested": true, "account": acct.Sqid, "app": app,
+ })
+}
+
// ---- trash and restore ------------------------------------------------------
// handleAPITrashApp answers `carlos apps delete`: it moves an app to the
diff --git a/internal/console/apirestart_test.go b/internal/console/apirestart_test.go
new file mode 100644
index 000000000..b099d58ba
--- /dev/null
+++ b/internal/console/apirestart_test.go
@@ -0,0 +1,177 @@
+package console
+
+// The `carlos restart` door (issue #113). What is pinned here, in the order it
+// would hurt if lost: the request actually lands in the bucket AND is
+// announced (a request nobody hears is a restart that never happens); a
+// stranger learns nothing from probing it; and the budget that keeps it from
+// becoming a restart loop is spent per person.
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// restartGenEtag is what every box watches: "" when nobody has ever asked.
+func restartGenEtag(t *testing.T, fx *consoleFixture, app string) string {
+ t.Helper()
+ _, etag, err := fx.rel.Get(context.Background(), release.RestartGenKey(release.OpsAccount, app))
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return ""
+ }
+ t.Fatalf("read restart generation: %v", err)
+ }
+ return etag
+}
+
+func restartFixture(t *testing.T, code string) (*consoleFixture, string) {
+ t.Helper()
+ fx := newConsoleFixture(t)
+ now := time.Now()
+ if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", now); err != nil {
+ t.Fatal(err)
+ }
+ // A MEMBER, not an owner: cycling your own app's processes is a member
+ // action, like promote below stable.
+ if err := fx.st.AddMember(OpsAccountID, "alice@keymail.dev", RoleMember, now); err != nil {
+ t.Fatal(err)
+ }
+ return fx, cliToken(t, fx, "alice@keymail.dev", code)
+}
+
+// TestAPIRestartTouchesTheGenerationAndAnnouncesIt is the whole door: the
+// object a box watches moves, the serial says "look now", and the answer names
+// the canonical account (the caller may have typed a name).
+func TestAPIRestartTouchesTheGenerationAndAnnouncesIt(t *testing.T) {
+ fx, token := restartFixture(t, "code-restart-roundtrip")
+ if got := restartGenEtag(t, fx, "titogo"); got != "" {
+ t.Fatalf("restart generation exists before any request: %q", got)
+ }
+
+ status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restart", token, "")
+ if status != http.StatusOK {
+ t.Fatalf("restart = %d %s, want 200", status, out)
+ }
+ var res struct {
+ Requested bool `json:"requested"`
+ Account string `json:"account"`
+ App string `json:"app"`
+ }
+ if err := json.Unmarshal(out, &res); err != nil {
+ t.Fatalf("restart body %s: %v", out, err)
+ }
+ if !res.Requested || res.Account != "bab" || res.App != "titogo" {
+ t.Fatalf("restart body = %+v, want requested for bab/titogo", res)
+ }
+ first := restartGenEtag(t, fx, "titogo")
+ if first == "" {
+ t.Fatal("no restart generation in the bucket after a 200 — the boxes will never hear about it")
+ }
+ if got := serial(t, fx); got != 1 {
+ t.Fatalf("serial after a restart request = %d, want 1 (otherwise boxes wait out the hourly floor)", got)
+ }
+
+ // A SECOND request must move the etag again: change detection is by etag,
+ // so a byte-identical rewrite would be a restart nobody performs.
+ status, out = postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restart", token, "")
+ if status != http.StatusOK {
+ t.Fatalf("second restart = %d %s", status, out)
+ }
+ if second := restartGenEtag(t, fx, "titogo"); second == first {
+ t.Fatalf("etag %q unchanged after a second request", second)
+ }
+ if got := serial(t, fx); got != 2 {
+ t.Fatalf("serial after two requests = %d, want 2", got)
+ }
+}
+
+// TestAPIRestartMemberGate: a non-member's refusal is byte-identical with an
+// unknown account's and an unknown app's — the same gate, in the same place,
+// as every other door here. Probing this one must not reveal that an app
+// exists, let alone restart it.
+func TestAPIRestartMemberGate(t *testing.T) {
+ fx, _ := restartFixture(t, "code-restart-gate-member")
+ eve := cliToken(t, fx, "eve@keymail.dev", "code-restart-gate-eve")
+
+ status, body := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restart", map[string]string{}, eve)
+ if status != http.StatusNotFound {
+ t.Fatalf("restart as non-member = %d, want 404", status)
+ }
+ if body != "404 page not found\n" {
+ t.Fatalf("restart as non-member body = %q, want the stdlib http.NotFound output", body)
+ }
+ // An unknown app in an account eve IS a member of would have to answer the
+ // same; here the account itself is not hers, and neither answer moved
+ // anything.
+ if got := restartGenEtag(t, fx, "titogo"); got != "" {
+ t.Fatal("a refused request touched the restart generation")
+ }
+ if got := serial(t, fx); got != 0 {
+ t.Fatalf("serial after a refused request = %d, want 0", got)
+ }
+}
+
+// TestAPIRestartNoTokenIs401: the token gate is first, before any name is
+// resolved.
+func TestAPIRestartNoTokenIs401(t *testing.T) {
+ fx, _ := restartFixture(t, "code-restart-notoken")
+ if status, _ := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restart", "", ""); status != http.StatusUnauthorized {
+ t.Fatalf("restart with no token = %d, want 401", status)
+ }
+}
+
+// TestAPIRestartIsRateLimited: this door's whole effect is to stop and start
+// processes, so the budget is what stands between a token and a restart loop
+// aimed at a live app. Keyed on the address, like every other write here.
+func TestAPIRestartIsRateLimited(t *testing.T) {
+ fx, token := restartFixture(t, "code-restart-limit")
+ fx.srv.cli.write = newRateLimiter(1, time.Minute)
+
+ if status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restart", token, ""); status != http.StatusOK {
+ t.Fatalf("first restart = %d %s, want 200", status, out)
+ }
+ status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restart", token, "")
+ if status != http.StatusTooManyRequests {
+ t.Fatalf("second restart on an exhausted budget = %d %s, want 429", status, out)
+ }
+ // A second TOKEN for the same person shares the budget: the principal is
+ // the person, not the terminal.
+ other := cliToken(t, fx, "alice@keymail.dev", "code-restart-limit-2")
+ if status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restart", other, ""); status != http.StatusTooManyRequests {
+ t.Fatalf("restart from a second token = %d %s, want 429", status, out)
+ }
+}
+
+// TestAPIRestartRefusesATrashedApp: the live-apps resolver is the gate, so an
+// app in the trash answers 404 like anything else that isn't there — there are
+// no processes to cycle, and this door must not become the way to ask whether
+// somebody's app was deleted.
+func TestAPIRestartRefusesATrashedApp(t *testing.T) {
+ fx, token := restartFixture(t, "code-restart-trashed")
+ if status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/delete", token, ""); status != http.StatusOK {
+ t.Fatalf("trash = %d %s", status, out)
+ }
+ if status, body := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restart", token, ""); status != http.StatusNotFound {
+ t.Fatalf("restart of a trashed app = %d %s, want 404", status, body)
+ }
+}
+
+// TestAPIRestartWithoutABucketIs503: the request IS a bucket object. A console
+// with nowhere to put it must say so rather than answer "requested" for
+// something no box will ever see.
+func TestAPIRestartWithoutABucketIs503(t *testing.T) {
+ fx, token := restartFixture(t, "code-restart-nobucket")
+ fx.srv.cfg.Releases = nil
+
+ status, out := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restart", token, "")
+ if status != http.StatusServiceUnavailable {
+ t.Fatalf("restart with no bucket = %d %s, want 503", status, out)
+ }
+}
diff --git a/internal/console/authcli.go b/internal/console/authcli.go
index 0f3351d02..29b0facbb 100644
--- a/internal/console/authcli.go
+++ b/internal/console/authcli.go
@@ -225,6 +225,11 @@ func (s *Server) registerCLIRoutes() {
s.mux.HandleFunc("POST /api/cli/apps/{account}/{app}/promote", s.handleAPIPromote)
s.mux.HandleFunc("POST /api/cli/apps/{account}/{app}/rollback", s.handleAPIRollback)
s.mux.HandleFunc("POST /api/cli/apps/{account}/{app}/releases", s.handleAPIShip)
+ // `carlos restart` (issue #113): a member's "cycle this app's processes"
+ // with no version and no config to push — the same write-door class as
+ // promote and rollback, and the only one whose subject is a running
+ // process rather than a pointer.
+ s.mux.HandleFunc("POST /api/cli/apps/{account}/{app}/restart", s.handleAPIRestart)
// The version-target, tags and releases-list API (apiversion.go,
// release versioning spec 2026-08-08). GET .../releases coexists with
// the POST above on the SAME path — a distinct method route, not a
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 8b6928fb7..a6612ebfc 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -719,6 +719,7 @@ type fastDeps struct {
restart func(string) error
passOpts []release.PassOption // fastPassOpts — every sweep pass carries all of them
fullPass func(ctx context.Context) // reconcile → adopt → config sync, or a test hook
+ restartLeg func(ctx context.Context) // the `carlos restart` check (issue #113), or a test hook
lastSerial int64
seeded bool
lastFull time.Time
@@ -855,6 +856,13 @@ func (d *fastDeps) backoffFor(host string) time.Duration { return d.step[host] }
// is deferred by NOT recording the acted serial, so the next tick past
// the window re-fires. Read errors are tolerated silently (one log
// line per state change, never per tick).
+// The restart leg (issue #113) rides that same move, AFTER the pass set:
+// a bump can carry a promote and a restart request at once, and cycling
+// processes before their new version and config land would cycle them onto
+// the old ones. It is a separate call rather than a fourth leg inside
+// fullPass because it is not release convergence — it restarts what is
+// already correct — and because it must still run on a box with no
+// CARLOS_SECRETS_KEY, which fullPass's config leg silently skips.
// 2. Fast sweep: fastMembers, and only when non-empty one PassFilter over
// exactly those hosts — an empty sweep costs zero store reads. After
// the sweep, members still unstaged back off; staged members clear
@@ -882,6 +890,9 @@ func fastTick(ctx context.Context, d *fastDeps, now time.Time) {
d.serialBroken = false
if doc.Serial != d.lastSerial && now.Sub(d.lastFull) >= fastFullClamp {
d.fullPass(ctx)
+ if d.restartLeg != nil {
+ d.restartLeg(ctx)
+ }
d.lastSerial = doc.Serial
d.lastFull = now
}
@@ -950,6 +961,13 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service) {
// Built once, used by both pass shapes below — see fastPassOpts for the
// DO-NOT list and why the stage requester is per-loop, not per-tick.
opts := fastPassOpts(dd, stageRequester(dd), restart)
+ // The restart check's memory (issue #113), seeded ONCE here — one read per
+ // app at arming and never again except on a serial move, which is what
+ // keeps an idle box at exactly one serial GET per tick. See restartSeed for
+ // why the baseline cannot wait for the first request to establish itself.
+ restartCache := map[[2]string]string{}
+ restartSeed(ctx, reg, store, dd, restartCache)
+ restartBounce := bounceFn(svc)
d := &fastDeps{
reg: reg, store: store, pub: pub, optDir: optDir,
restart: restart, passOpts: opts,
@@ -970,6 +988,17 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service) {
}
}
},
+ // The restart leg: this process's own children bounced here, everything
+ // unit-shaped relayed to a root pass. Deliberately NOT inside fullPass
+ // above — see fastTick's comment for both reasons.
+ restartLeg: func(ctx context.Context) {
+ routes, err := reg.List()
+ if err != nil {
+ log.Printf("agent: restart check: list routes: %v", err)
+ return
+ }
+ restartRequestLeg(ctx, routes, store, dd, restartBounce, restartCache)
+ },
}
log.Printf("agent: fast converge loop armed (2s + jitter: serial poll + canary/* + unstaged routes)")
for {
@@ -1415,6 +1444,10 @@ func statusLoop(reg *registry.Registry) {
// carlos-adopt.path is a box whose unit-backed config silently
// waits for the hourly timer (issue #44).
ConfigPathRunning: configPathActive(),
+ // A third, for the restart trigger: an enabled-but-unstarted
+ // carlos-restart.path is a box where `carlos restart` answers
+ // "requested" and nothing cycles until the hourly pass (issue #113).
+ RestartPathRunning: restartPathActive(),
// ExpiredCanaries answers spec §5 for this box's own routes; see
// countExpiredCanaries and the canaryStore/canaryPub comment above.
ExpiredCanaries: countExpiredCanaries(ctx, routes, canaryStore, canaryPub, time.Now()),
@@ -1689,11 +1722,22 @@ func openDeployStore(ctx context.Context) (release.Store, error) {
// seconds of an unrelated write, and would fire the canary-TTL nag per
// config write instead of hourly — the "hourly root pass alone" split
// release/adopt.go and the nag comment below both depend on.
+//
+// -restart-only runs the RESTART-REQUEST half alone (issue #113): compare each
+// app's restart generation against this box's marks and cycle the ones that
+// moved, nothing else. That is what carlos-restart.service execs, and it is
+// separated from -config-only for the same reason -config-only is separated
+// from the full pass: the trigger behind it is a per-request path unit, and a
+// request to cycle one app's processes must not stage a release, run the
+// capability converge, or nag about canaries. It deliberately needs NO release
+// key and NO secrets key — restarting verifies no signature and renders no
+// config — so a box missing either still honours it.
func RunAdopt(args []string) {
fs := flag.NewFlagSet("adopt", flag.ExitOnError)
regPath := fs.String("registry", filepath.Join(dataDir(), "registry.db"), "registry database")
configOnly := fs.Bool("config-only", false, "sync unit-backed config and restart changed routes only: no release adoption, no canary nag")
stageOnly := fs.Bool("stage-only", false, "adopt releases without the canary nag: the agent-requested pass (carlos-stage.service)")
+ restartOnly := fs.Bool("restart-only", false, "cycle the routes of apps whose restart generation moved, and nothing else: the agent-requested pass (carlos-restart.service)")
_ = fs.Parse(args)
reg, err := registry.Open(*regPath)
if err != nil {
@@ -1717,6 +1761,24 @@ func RunAdopt(args []string) {
// construction. It is handed the same callback anyway because there should
// be ONE restart dispatch on this pass, not two that could drift.
restart := rootRestartRoute(reg)
+ // The restart-request half, and the ONE place it dispatches from —
+ // including for exec-backed routes, whose children belong to the agent:
+ // rootRestartRoute's try-restart arm reaches a carlos-app@ unit, and an
+ // exec child that this pass cannot cycle is cycled by nothing else, which
+ // is a real limit of the -restart-only path recorded in restartPass.
+ //
+ // -restart-only returns here: no config sync, no capability converge, no
+ // release.Pass, no nag, and — above all — no release public key, which is
+ // parsed below and is fatal when unusable. A restart request must not fail
+ // on a key it never needs.
+ if *restartOnly {
+ routes, err := reg.List()
+ if err != nil {
+ log.Fatalf("adopt: list routes: %v", err)
+ }
+ restartPass(ctx, routes, store, restartMarksPath(dataDir()), restart)
+ return
+ }
// Parsed here, before config sync, exactly as the full pass always has
// (a box with an unusable release key must die before it converges
// anything), but SKIPPED entirely for -config-only: that mode never
@@ -1745,10 +1807,28 @@ func RunAdopt(args []string) {
if *configOnly {
// Everything below is release adoption and its reporting — the half
// carlos-config-sync.service must never reach, however often a
- // config write triggers it.
+ // config write triggers it. The restart half is below for the same
+ // reason: a config write is not a restart request, and running the
+ // comparison here would only move it earlier by minutes at the cost of
+ // firing on a trigger that has nothing to do with it.
return
}
+ // The restart-request half again (issue #113), here as the FLOOR under
+ // carlos-restart.path: that relay can drop an event that lands while the
+ // service is mid-run, exactly as the config one can, and a dropped restart
+ // with no floor would be a request that never happens at all.
+ //
+ // BEFORE release.Pass deliberately. That call is fatal on error, so a box
+ // with an unstageable release would otherwise never reach this line — and
+ // a wedged instance is most worth cycling precisely on a box where
+ // something else is already wrong.
+ if routes, err := reg.List(); err != nil {
+ log.Printf("adopt: list routes for the restart pass: %v", err)
+ } else {
+ restartPass(ctx, routes, store, restartMarksPath(dataDir()), restart)
+ }
+
// Step 2 (spec §4d): the grant-drift capability converge, BETWEEN config
// sync and release adoption. Before release.Pass because it repairs the
// shape of what is ALREADY live — a revoked grant, a deleted drop-in, a
diff --git a/internal/host/bootstrap.go b/internal/host/bootstrap.go
index 1baf02394..f5bb7765a 100644
--- a/internal/host/bootstrap.go
+++ b/internal/host/bootstrap.go
@@ -58,14 +58,16 @@ const tmpfilesConf = "d /run/carlos 2750 " + serviceUser + " " + socketsGroup +
"d /run/carlos/apps 0750 " + serviceUser + " " + serviceUser + " -\n"
// enabledUnits are what a live bootstrap turns on: the always-on front door,
-// the hourly adoption timer, and the two path units that let the sandboxed
+// the hourly adoption timer, and the three path units that let the sandboxed
// agent ask a root pass for the halves it cannot do itself: carlos-adopt.path
-// (config fast convergence, issue #44) and carlos-stage.path (release staging
-// — the agent sees a promotion in seconds but cannot write /opt). The litestream + restore-verify units install
+// (config fast convergence, issue #44), carlos-stage.path (release staging
+// — the agent sees a promotion in seconds but cannot write /opt) and
+// carlos-restart.path (a member's `carlos restart`, issue #113 — the agent
+// bounces its own exec children but cannot cycle a unit). The litestream + restore-verify units install
// everywhere but stay inert until an operator stages /data/litestream.env and
// generates /etc/litestream.yml (`carlos ops litestream`) — a box with no
// replication bucket is unaffected.
-var enabledUnits = []string{"carlos-edge.service", "carlos-adopt.timer", "carlos-adopt.path", "carlos-stage.path"}
+var enabledUnits = []string{"carlos-edge.service", "carlos-adopt.timer", "carlos-adopt.path", "carlos-stage.path", "carlos-restart.path"}
// systemctlPlan is the exact systemctl sequence a live bootstrap runs — and
// the same sequence a staging run (-root <dir>) PRINTS instead of running, so
diff --git a/internal/host/host_test.go b/internal/host/host_test.go
index e432768fc..52c7d1a4e 100644
--- a/internal/host/host_test.go
+++ b/internal/host/host_test.go
@@ -33,6 +33,8 @@ var expectedUnits = []string{
"carlos-config-sync.service",
"carlos-edge.service",
"carlos-litestream.service",
+ "carlos-restart.path",
+ "carlos-restart.service",
"carlos-restore-verify.service",
"carlos-restore-verify.timer",
"carlos-stage.path",
diff --git a/internal/host/probes.go b/internal/host/probes.go
index 6a64ce1c4..5264137f3 100644
--- a/internal/host/probes.go
+++ b/internal/host/probes.go
@@ -62,6 +62,25 @@ func configPathActive() *bool {
return parseIsActive(string(out))
}
+// restartPathUnit is the unit whose liveness IS "a member's `carlos restart`
+// reaches this box's units in seconds" (issue #113): the path unit watching
+// /data/restart-requested, which the agent touches and which starts
+// carlos-restart.service. Installed and armed by bootstrap alongside the other
+// two (see enabledUnits).
+const restartPathUnit = "carlos-restart.path"
+
+// restartPathActive asks the same question configPathActive asks, about a unit
+// with the same silent failure mode — and one worse consequence. An unarmed
+// config trigger costs an env var an hour; an unarmed restart trigger means
+// someone typed `carlos restart` at a wedged instance, was told "requested",
+// and will stand there watching a process that will not cycle until the hourly
+// pass comes round. Nothing inside the agent can see the difference: it writes
+// its marker either way.
+func restartPathActive() *bool {
+ out, _ := exec.Command("systemctl", "is-active", restartPathUnit).Output()
+ return parseIsActive(string(out))
+}
+
// parseIsActive maps systemd's is-active vocabulary onto the tri-state the
// status page publishes. "unknown" — systemd's answer when it has no such unit
// loaded — is a genuine false: a box with no adoption timer adopts nothing.
diff --git a/internal/host/restart.go b/internal/host/restart.go
new file mode 100644
index 000000000..7ebe61726
--- /dev/null
+++ b/internal/host/restart.go
@@ -0,0 +1,650 @@
+// The box half of `carlos restart` (issue #113): when an app's
+// restart-generation object (release.RestartGenKey) MOVES, cycle that app's
+// live processes on this box. Pull-based like everything else here — no push
+// channel, no box access, no version or config change involved.
+//
+// IT RIDES THE SERIAL, and that is the load-bearing design decision. The
+// console door bumps control/serial after touching the object, exactly as
+// every other console write does, and the agent's existing 2s serial poll
+// (fastTick) already turns a bump into the full convergence pass set. The
+// restart check is one more leg of THAT — so a box with nothing happening
+// pays its one serial GET per tick and NOT ONE READ MORE. An earlier revision
+// polled every app's restart generation on its own 3s timer; measured, that
+// was ~575k needless GETs per box per day to notice an event that arrives a
+// handful of times a month. A verb that costs the fleet a permanent poll storm
+// to exist is not a verb worth having.
+//
+// Direct-bucket touches that never bump the serial (a script writing the
+// object straight into the bucket) converge on the hourly root pass instead —
+// the same floor every other direct-bucket write already relies on.
+//
+// The work is split across two processes, for the reason config fast
+// convergence is split the same way:
+//
+// - the AGENT (carlos-edge.service, User=carlos, ProtectSystem=strict) owns
+// the EXEC-backed instances outright, so its half is the etag comparison,
+// a bounce of its own children, and one marker file for the rest.
+// systemctl cannot reach a child of another process, so a root-only
+// dispatch would leave every warm exec instance — the commonest shape on
+// the box, and the one a wedged instance usually is — serving on,
+// unrestarted.
+// - the ROOT pass (carlos adopt, out of carlos-restart.service via
+// carlos-restart.path, or the hourly timer) is the only one that can
+// restart a systemd unit — the carlos-app@ tenants and the bespoke units
+// addr-backed routes name (issue #110's consoles). Its half is the
+// comparison against a persisted, per-app high-water mark and the
+// dispatch.
+//
+// SEED-DON'T-REPLAY governs both halves, PER APP. An app this box has not
+// recorded yet is recorded and NOT restarted, whether its object exists or
+// not; only movement away from a value this box has already written counts.
+// The absent case gets its own recorded sentinel precisely so that an app's
+// first ever request (absent -> present) is still movement rather than a first
+// sight — that distinction is the difference between the verb working and the
+// verb cycling a box's whole route table the day a marks file goes missing.
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "os"
+ "path/filepath"
+ "syscall"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// restartApps lists the distinct (account, app) pairs with at least one
+// restartable route on this box — the apps whose restart generation this box
+// reads when a serial bump (or an hourly pass) gives it a reason to.
+//
+// The membership predicate is release.Adoptable, deliberately the SAME one the
+// adopt pass moves and the fast sweep counts: a route a pass would not adopt
+// is not one whose process this feature has any business cycling, and a second
+// copy of that judgement here is exactly the drift that would let the two
+// disagree. It admits addr-backed routes that name their own unit (issue
+// #110's consoles) — which is half of why #113 exists — because Adoptable
+// filters on lifecycle and app, never on backing.
+func restartApps(routes []registry.Route) [][2]string {
+ seen := map[[2]string]bool{}
+ var out [][2]string
+ for _, rt := range routes {
+ if !release.Adoptable(rt) {
+ continue
+ }
+ k := [2]string{release.RouteAccount(rt), rt.App}
+ if seen[k] {
+ continue
+ }
+ seen[k] = true
+ out = append(out, k)
+ }
+ return out
+}
+
+// restartGenPass is the agent's comparison, split out for tests: read the
+// restart-generation etag of every app this box serves and report WHICH of
+// them moved. Same semantics — and the same reasons — as genPollPass:
+//
+// - First sight of an app records silently and asks for nothing. A box
+// coming up must not restart everything it serves because it has never
+// seen these etags before; that is the seed-don't-replay rule, and it is
+// the whole safety property of this feature. restartSeed runs one such
+// silent pass at arming, so the FIRST request after a boot is acted on
+// rather than swallowed as the baseline.
+// - An unchanged etag does nothing.
+// - Absent that stays absent does nothing, FOREVER: an app nobody has ever
+// restarted this way has no object and never will until someone asks.
+// That is the ordinary state of almost every app, not a fault.
+// - Any movement asks: the etag changed, or absent -> present (an app's
+// first ever restart request). present -> absent should not happen
+// (restart generations are only created and overwritten, never deleted),
+// but it counts as movement too — one extra root pass is cheap, and
+// pretending we did not see it would be a lie about the only thing this
+// function reports.
+// - The moved apps come back as a LIST, but the root request is one marker
+// touch however long that list is: the root pass re-reads every app's etag
+// itself and dispatches per app, so asking once per moved app would only
+// ask the same pass to run several times. The list is for the caller's own
+// half — which of ITS exec children to bounce.
+// - A read error that is not "not found" leaves that app's cache entry
+// exactly as it was and keeps evaluating the rest — the errors come back
+// for the caller to aggregate into ONE log line per pass. Clearing the
+// entry would turn a recovered blip into a bogus first sight or a bogus
+// restart.
+// - Apps that vanish from the route set lose their cache entries: a re-added
+// app is a first sight again, which is right, because a route that has
+// just been (re-)minted is starting its process fresh anyway.
+//
+// The cache is mutated in place and lives for the life of the agent process
+// (fastConvergeLoop owns it, restartSeed fills it), which is the whole point:
+// there is nowhere else to remember what "changed" means between two serial
+// moves that may be days apart.
+func restartGenPass(ctx context.Context, routes []registry.Route, store release.Store, cache map[[2]string]string) (moved [][2]string, readErrs []error) {
+ apps := restartApps(routes)
+ live := make(map[[2]string]bool, len(apps))
+ for _, k := range apps {
+ live[k] = true
+ _, etag, err := store.Get(ctx, release.RestartGenKey(k[0], k[1]))
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ etag = genAbsent
+ case err != nil:
+ readErrs = append(readErrs, fmt.Errorf("%s/%s: %w", k[0], k[1], err))
+ continue // cache entry untouched: this app is unknown this tick, not changed
+ }
+ prev, seen := cache[k]
+ cache[k] = etag
+ if seen && prev != etag {
+ moved = append(moved, k)
+ }
+ }
+ for k := range cache {
+ if !live[k] {
+ delete(cache, k)
+ }
+ }
+ return moved, readErrs
+}
+
+// bounceExecRoutes cycles the agent's OWN children for the apps whose restart
+// generation just moved, and returns how many it actually stopped. This is the
+// half no root pass can perform: an exec instance is a child of THIS process,
+// and systemctl has no name for it.
+//
+// The route filter is the complement of what rootRestartRoute's systemctl arms
+// can reach — no named unit, no unit backing — so the two halves never cycle
+// the same process twice.
+//
+// Coverage is NOT total, and the gap is worth naming: a route that is
+// activator-backed (Backing "", an exec child) but ALSO carries a Unit stamp
+// is skipped here (it names a unit) while the root half try-restarts that
+// named unit, which is not the process the activator is running. Such a route
+// is a misconfiguration — `carlos route -unit` is for addr-backed tenants
+// whose process an operator owns — but nothing in this file repairs it, and
+// its exec child is cycled by NEITHER half until somebody clears the stamp.
+//
+// HIBERNATE-flagged routes are included, and that is not a contradiction of
+// "never wake a sleeper": Bounce provably cannot start anything (it is a no-op
+// unless the instance is AWAKE), so a sleeping instance is untouched and picks
+// its next start up naturally, while a WARM hibernator — which is what a
+// wedged instance usually is — gets exactly the cycle that was asked for. This
+// is release.RestartHibernating's argument, in the one process that can act on
+// it.
+//
+// A nil bounce (no activator armed on this box) is a working no-op: there are
+// no children to cycle, and the unit half still runs.
+func bounceExecRoutes(routes []registry.Route, moved [][2]string, bounce func(host string) bool) int {
+ if bounce == nil || len(moved) == 0 {
+ return 0
+ }
+ want := make(map[[2]string]bool, len(moved))
+ for _, k := range moved {
+ want[k] = true
+ }
+ n := 0
+ for _, rt := range routes {
+ if !release.Adoptable(rt) || rt.Unit != "" || rt.Backing == registry.BackingUnit {
+ continue
+ }
+ if !want[[2]string{release.RouteAccount(rt), rt.App}] {
+ continue
+ }
+ if bounce(rt.Host) {
+ n++
+ log.Printf("agent: restart requested: bounced exec instance %s", rt.Host)
+ }
+ }
+ return n
+}
+
+// restartRequestMarker is the file the agent's restart leg touches to ask a
+// root pass to run now: carlos-restart.path watches it (PathChanged) and starts
+// carlos-restart.service (`carlos adopt -restart-only`), which owns the half
+// this unprivileged process cannot do — restarting units — and only that half.
+//
+// A sibling of configSyncMarker rather than a reuse of it, because the two
+// relays land on different services: config's target is `carlos adopt
+// -config-only`, which renders /etc/carlos/apps and restarts only the routes
+// whose env file actually changed. A restart request changes no file, so that
+// pass would find nothing to do and cycle nothing at all.
+//
+// It lives in the data dir, deliberately outside the *.config.env namespace
+// reapConfigEnv sweeps — the reaper matches that suffix alone and must go on
+// ignoring this.
+const restartRequestMarker = "restart-requested"
+
+// requestRootRestart touches the marker. The content is a timestamp only so an
+// operator reading /data can see when the last request went out; systemd
+// watches the path, not the bytes. Errors are the caller's to log and swallow:
+// a request that cannot be written costs latency, not correctness, because
+// carlos-adopt.timer's hourly pass runs the same comparison underneath it.
+func requestRootRestart(dd string) error {
+ return os.WriteFile(filepath.Join(dd, restartRequestMarker),
+ []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644)
+}
+
+// restartMarksFile is the ROOT pass's memory: the last restart generation it
+// has already acted on, per app. It lives in the data dir beside the bake
+// clock, in the same JSON + temp-file-rename shape and for the same
+// cross-uid reason (see restartMarks.save).
+const restartMarksFile = "restart-marks.json"
+
+func restartMarksPath(dd string) string { return filepath.Join(dd, restartMarksFile) }
+
+// restartMarks is that memory. Apps is keyed "<account>|<app>" — "|" is
+// outside every character class a sqid or an app name allows, the same
+// separator argument internal/release's bake clock makes for its own keys.
+//
+// A value is either the etag this box last acted on OR restartAbsent, the
+// recorded "this app has no restart generation and I have looked". On disk:
+//
+// {
+// "schema": 1,
+// "apps": {
+// "bab|keymail": "\"9f86d0…\"",
+// "bab|titogo": "\u0000absent"
+// }
+// }
+//
+// The absent sentinel is what makes per-app seed-don't-replay possible. Without
+// it "no entry" has to mean both "never looked" and "looked, nothing there",
+// and one of those two has to be wrong: read as movement, a box gaining a new
+// route for a previously-restarted app cycles it for a request served weeks
+// ago (and a box whose registry lands after its marks file cycles EVERY route
+// at once); read as a first sight, an app's first ever request is silently
+// dropped. Recording the absence tells them apart — a later object sighting is
+// movement from a value this box wrote, which is exactly what it is.
+//
+// Entries are never pruned when a route leaves the box. A vanished app's mark
+// costs one map entry; DROPPING it would make the app's return look like a
+// brand-new object and cycle it once for a request that was already served.
+type restartMarks struct {
+ Schema int `json:"schema"` // 1
+ Apps map[string]string `json:"apps"`
+}
+
+// restartAbsent is the marks value for an app this box has SEEN and found to
+// have no restart-generation object at all. It is deliberately the SAME value
+// the in-memory sweep uses (genAbsent), so the agent's memory and the root
+// pass's memory cannot drift apart on what "absent" is, and it is safe for the
+// same reason: no store produces an etag containing a NUL byte (S3's are
+// hex/quoted, DirStore's is hex). JSON renders it "\u0000absent".
+const restartAbsent = genAbsent
+
+func restartMarkKey(acct, app string) string { return acct + "|" + app }
+
+// lockRestartMarks takes an exclusive flock for the whole of one restartPass.
+// Three root passes can run this — carlos-restart.service (the relay),
+// carlos-stage.service and carlos-adopt.service — and systemd will happily
+// start them at the same second. Unserialized, two passes read the same marks,
+// both dispatch (one request, two try-restarts, on a verb whose entire job is
+// stopping processes) and then both write, so whichever finishes last silently
+// reverts the other's advance and the NEXT pass restarts everything a third
+// time.
+//
+// The lock lives in a sibling <marks>.lock rather than on the marks file
+// itself, because the marks file is replaced by rename: a flock is held on an
+// inode, and the winner's rename would leave the loser holding a lock on a file
+// that is no longer the marks file.
+//
+// Opening it is ALSO the writability test the dispatch is gated on (see
+// restartPass): it is created in the same directory, with the same permissions
+// the marks save needs, so a full or read-only /data fails here — before
+// anything is cycled — instead of after, which is the difference between one
+// logged refusal and an hourly restart of every app on the box forever.
+//
+// syscall.Flock is available on both linux and darwin, which is the whole
+// build matrix (CI cross-compiles darwin because the CLI runs on Macs).
+func lockRestartMarks(path string) (unlock func(), err error) {
+ f, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR, 0o644)
+ if err != nil {
+ return nil, err
+ }
+ if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
+ f.Close()
+ return nil, err
+ }
+ return func() {
+ syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+ f.Close()
+ }, nil
+}
+
+// loadRestartMarks reads the marks file. `known` is the seed-don't-replay
+// switch and the only subtle thing here: it is FALSE when the file is missing,
+// unreadable, or does not parse — three different accidents with one correct
+// response, which is to record what the bucket currently says and restart
+// NOTHING this pass.
+//
+// That direction is the opposite of the bake clock's ("an unreadable clock
+// fails toward baking"), and deliberately so: baking is the cautious act there,
+// while restarting is the DISRUPTIVE act here. The cost of failing this way is
+// that a box whose marks file was just wiped can drop ONE restart request —
+// logged, and re-runnable by the person who typed it. The cost of failing the
+// other way is a box that cycles every app it serves the moment its state file
+// goes missing.
+func loadRestartMarks(path string) (marks *restartMarks, known bool) {
+ m := &restartMarks{Schema: 1, Apps: map[string]string{}}
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ log.Printf("adopt: restart marks %s unreadable (%v) — re-baselining, no restarts this pass", path, err)
+ }
+ return m, false
+ }
+ if err := json.Unmarshal(raw, m); err != nil {
+ log.Printf("adopt: restart marks %s do not parse (%v) — re-baselining, no restarts this pass", path, err)
+ return &restartMarks{Schema: 1, Apps: map[string]string{}}, false
+ }
+ if m.Apps == nil {
+ m.Apps = map[string]string{}
+ }
+ m.Schema = 1
+ return m, true
+}
+
+// save persists the marks via temp-file-in-the-same-dir + rename, the
+// co-written-file pattern release.FirstSeen.Save documents at length: /data is
+// written by BOTH uids on this box (carlos for the agent's seed below, root
+// for this pass), and an in-place O_TRUNC write needs write permission on the
+// existing file, so whichever uid created it would lock the other out forever.
+// rename(2) only needs write permission on the directory, which both have. The
+// temp name is minted fresh per attempt so a crash between write and rename
+// can never leave cross-uid residue for the next save to trip over, and the
+// 0600 CreateTemp mints is widened to 0644 BEFORE the rename so the other uid
+// can still read it.
+func (m *restartMarks) save(path string) error {
+ raw, err := json.MarshalIndent(m, "", " ")
+ if err != nil {
+ return err
+ }
+ raw = append(raw, '\n')
+ f, err := os.CreateTemp(filepath.Dir(path), "restart-marks-*.json")
+ if err != nil {
+ return err
+ }
+ tmp := f.Name()
+ _, writeErr := f.Write(raw)
+ closeErr := f.Close()
+ if writeErr != nil {
+ os.Remove(tmp)
+ return writeErr
+ }
+ if closeErr != nil {
+ os.Remove(tmp)
+ return closeErr
+ }
+ if err := os.Chmod(tmp, 0o644); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ if err := os.Rename(tmp, path); err != nil {
+ os.Remove(tmp) // don't leave a stray temp file behind
+ return err
+ }
+ return nil
+}
+
+// seedRestartMarks writes the baseline the ROOT pass compares against, ONCE,
+// from the agent's arming sweep — create-only (O_EXCL), so it can never
+// overwrite a mark the root pass has already advanced.
+//
+// It exists to close a real hole in the handover between the two halves. The
+// agent seeds its own baseline as it arms; the root pass would otherwise not
+// seed until it next ran, and a restart requested inside that window would
+// arrive at a pass with no baseline — which, by the rule above, records and
+// restarts nothing. The request would be silently dropped, having told its
+// caller "requested". Creating the file here shrinks that window to the moment
+// the agent starts, where the box has just restarted everything anyway.
+//
+// An EMPTY cache writes nothing at all. A sweep that saw no adoptable routes
+// is not a baseline — most often it is a box whose registry has not landed yet
+// (a fresh machine, or one restoring /data from litestream) — and a file
+// claiming "this box knows about no apps" is precisely the file that makes the
+// root pass treat every route it later gains as movement, and cycle the lot in
+// one pass.
+//
+// Absent objects ARE recorded, with restartAbsent: "looked, nothing there" is
+// the fact that makes an app's first ever request movement rather than a first
+// sight.
+func seedRestartMarks(path string, cache map[[2]string]string) error {
+ if len(cache) == 0 {
+ return nil // nothing was seen; nothing may be claimed
+ }
+ if _, err := os.Stat(path); err == nil {
+ return nil // the root pass (or an earlier agent) already has a baseline
+ } else if !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ m := &restartMarks{Schema: 1, Apps: map[string]string{}}
+ for k, etag := range cache {
+ m.Apps[restartMarkKey(k[0], k[1])] = etag
+ }
+ raw, err := json.MarshalIndent(m, "", " ")
+ if err != nil {
+ return err
+ }
+ raw = append(raw, '\n')
+ f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
+ if err != nil {
+ if errors.Is(err, os.ErrExist) {
+ return nil // lost the race to the root pass; its mark is the better one
+ }
+ return err
+ }
+ _, writeErr := f.Write(raw)
+ if err := f.Close(); err != nil && writeErr == nil {
+ writeErr = err
+ }
+ return writeErr
+}
+
+// restartPass is the ROOT half: for every app on this box whose restart
+// generation has moved since the mark this box recorded, cycle that app's
+// routes. Called from RunAdopt — on the marker-triggered `-restart-only` run
+// (seconds after the request) and on the ordinary pass (the hourly floor
+// underneath the relay, which can drop an event that lands while the service
+// is mid-run).
+//
+// The rules, each a deliberate choice:
+//
+// - ZERO adoptable routes ends the pass before anything is read or written.
+// A box whose registry has not landed yet (fresh, or restoring /data) must
+// not be able to record "I know about no apps" — that file is what turns
+// every route the box later gains into movement, and cycles the whole
+// table in one pass.
+// - An app this box has NOT recorded yet is recorded and NOT cycled,
+// whether its object exists or not. Per-app seed-don't-replay: the marks
+// file is this box's memory of what it has acted on, and a request it
+// never saw is not one it owes anybody.
+// - An app with NO restart generation records restartAbsent — the sentinel
+// that makes its FIRST ever request (absent -> present) movement rather
+// than another first sight. Absent is legal forever; unrecorded absence is
+// what was not.
+// - An unknown baseline (see loadRestartMarks) records every app and
+// dispatches nothing — and if any app could not be READ, records nothing
+// at all: a baseline missing the apps a transient 503 hid is a baseline
+// that turns those apps into movement on the next pass. A dropped request
+// is recoverable (run the verb again); a restart nobody asked for is not.
+// - HIBERNATING routes are skipped outright and not marked as a failure.
+// Restarting a process that is not running is meaningless, waking it to
+// restart it is worse (it would defeat hibernation for no gain), and it
+// will exec fresh on its next wake regardless — which is the restart. A
+// WARM hibernating exec instance is not left serving, though: the agent's
+// own bounce (bounceExecRoutes) reaches it, in the one process that can.
+// - A dispatch FAILURE still advances the mark, and logs. A restart request
+// is a point-in-time event, not a desired state: a named unit that exits
+// non-zero (a wrong unit name, a service that will not come back) would
+// otherwise be retried on every pass, forever, for a request somebody made
+// once. The journal line is the report; `carlos restart` again is the
+// retry.
+// - The mark is written only AFTER the dispatch, so a pass that dies
+// mid-cycle retries rather than swallowing the request.
+// - The whole pass holds an exclusive lock (lockRestartMarks), because three
+// root units can run it at once; failing to TAKE that lock dispatches
+// nothing, since the lock file's creation is also the proof that the marks
+// can be saved afterwards. A pass that cycles processes and then cannot
+// record what it did would cycle them again every hour, forever.
+//
+// Holds, bake windows and the release ladder take no part in any of this:
+// nothing about which version runs changes here, so there is nothing for them
+// to gate.
+func restartPass(ctx context.Context, routes []registry.Route, store release.Store, marksPath string, restart func(host string) error) {
+ apps := restartApps(routes)
+ if len(apps) == 0 {
+ return // nothing to converge, and nothing this box may claim to know
+ }
+ unlock, err := lockRestartMarks(marksPath)
+ if err != nil {
+ // No lock means either a concurrent pass we cannot serialize against or
+ // a /data we cannot write. Both make dispatching the wrong move: the
+ // first double-restarts, the second restarts hourly and forever because
+ // the mark can never advance.
+ log.Printf("adopt: restart pass: cannot lock %s.lock (%v) — nothing cycled this pass", marksPath, err)
+ return
+ }
+ defer unlock()
+
+ marks, known := loadRestartMarks(marksPath)
+ changed := false
+ readFailed := false
+ for _, k := range apps {
+ _, etag, err := store.Get(ctx, release.RestartGenKey(k[0], k[1]))
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ etag = restartAbsent
+ case err != nil:
+ log.Printf("adopt: restart generation %s/%s: %v (skipped this pass)", k[0], k[1], err)
+ readFailed = true
+ continue
+ }
+ key := restartMarkKey(k[0], k[1])
+ prev, seen := marks.Apps[key]
+ if !known || !seen {
+ // SEED. Either this box's whole memory is untrustworthy, or this is
+ // the first time it has looked at this app — a route it has just
+ // gained, whose process is starting fresh anyway. Record, cycle
+ // nothing.
+ if !seen || prev != etag {
+ marks.Apps[key] = etag
+ changed = true
+ }
+ continue
+ }
+ if prev == etag {
+ continue
+ }
+ dispatched := 0
+ for _, rt := range routes {
+ if !release.Adoptable(rt) || rt.App != k[1] || release.RouteAccount(rt) != k[0] {
+ continue
+ }
+ if rt.Hibernate {
+ log.Printf("adopt: restart requested for %s/%s: %s is hibernating — left asleep (it execs fresh on its next wake)",
+ k[0], k[1], rt.Host)
+ continue
+ }
+ if err := restart(rt.Host); err != nil {
+ log.Printf("adopt: restart requested for %s/%s: %s did not cycle: %v", k[0], k[1], rt.Host, err)
+ continue
+ }
+ dispatched++
+ }
+ // One line per APP, not per route, and it says DISPATCHED rather than
+ // cycled: an exec-backed route reaching rootRestartRoute lands on a
+ // try-restart of a carlos-app@ unit that does not exist — a deliberate
+ // no-op (the agent bounced that child itself) which this pass cannot
+ // tell apart from a real one. Claiming "cycled" for it would be the
+ // kind of lie issue #110 was made of.
+ log.Printf("adopt: restart requested for %s/%s: dispatched to %d route(s)", k[0], k[1], dispatched)
+ marks.Apps[key] = etag
+ changed = true
+ }
+ if !known && readFailed {
+ // A baseline that omits the apps a blip hid is worse than no baseline:
+ // this pass would write "known", and the next one would read those
+ // missing apps as movement and cycle them. Leave the file as it is and
+ // re-baseline on a pass that can read everything.
+ log.Printf("adopt: restart pass: baselining deferred — some apps could not be read this pass")
+ return
+ }
+ // A first, unknown baseline is written even when nothing changed: that
+ // write is what makes the NEXT pass a known one, and until it lands every
+ // pass would re-baseline and drop whatever it was asked to do.
+ if changed || !known {
+ if err := marks.save(marksPath); err != nil {
+ log.Printf("adopt: save restart marks %s: %v (the next pass may repeat a restart)", marksPath, err)
+ }
+ }
+}
+
+// restartRequestLeg is the agent's half of one serial move: compare each app's
+// restart generation against the in-memory cache, bounce this process's own
+// exec children for the apps that moved, and touch the marker that asks a root
+// pass to cycle everything unit-shaped. Reports whether it asked.
+//
+// It runs ONLY when the serial moved (fastTick), which is what makes the
+// feature free at rest: the console bumps the serial after touching the
+// object, so every request is announced, and an idle box reads no restart
+// generation at all. There is no loop of its own and there must not be one —
+// see this file's package comment for what the 3s version of this cost.
+//
+// The bounce goes first, and not only for tidiness: it is in-process and
+// instant, while the marker write can block on a busy /data, and the children
+// bounced here are the ones cycled here or nowhere.
+func restartRequestLeg(ctx context.Context, routes []registry.Route, store release.Store, dd string,
+ bounce func(host string) bool, cache map[[2]string]string) bool {
+ moved, readErrs := restartGenPass(ctx, routes, store, cache)
+ if len(readErrs) > 0 {
+ // ONE line per pass, never one per app: the count carries the scale,
+ // the first error the cause — they are the same cause in practice.
+ log.Printf("agent: restart check: %d apps unreadable: %v", len(readErrs), readErrs[0])
+ }
+ if len(moved) == 0 {
+ return false
+ }
+ bounceExecRoutes(routes, moved, bounce)
+ if err := requestRootRestart(dd); err != nil {
+ log.Printf("agent: request root restart pass: %v", err)
+ return false
+ }
+ log.Printf("agent: restart requested for %d app(s) on this box, asked for a root restart pass", len(moved))
+ return true
+}
+
+// restartSeed is the ONE sweep this agent makes on its own initiative: at
+// arming, read each app's restart generation once, silently, so the cache the
+// leg compares against exists before the first serial move — and hand the same
+// sightings to seedRestartMarks so the root pass has a baseline too.
+//
+// Without it the first request after a boot is the sweep that establishes the
+// baseline, and is therefore the one request that does nothing: the person who
+// typed `carlos restart` would have to type it twice, having been told
+// "requested" the first time. One read per app per agent start is the whole
+// cost, and it is paid where a box has just restarted everything anyway.
+//
+// A read error leaves that app out of the cache (its next sighting is a first
+// sight — safe) and blocks the marks seed entirely, for the reason
+// seedRestartMarks and restartPass both give: a baseline with holes in it is
+// the thing that manufactures restarts.
+func restartSeed(ctx context.Context, reg *registry.Registry, store release.Store, dd string, cache map[[2]string]string) {
+ routes, err := reg.List()
+ if err != nil {
+ log.Printf("agent: restart baseline: list routes: %v (the first request may need a second try)", err)
+ return
+ }
+ _, readErrs := restartGenPass(ctx, routes, store, cache)
+ if len(readErrs) > 0 {
+ log.Printf("agent: restart baseline: %d apps unreadable: %v", len(readErrs), readErrs[0])
+ return
+ }
+ if err := seedRestartMarks(restartMarksPath(dd), cache); err != nil {
+ log.Printf("agent: seed restart marks: %v", err)
+ }
+}
diff --git a/internal/host/restart_test.go b/internal/host/restart_test.go
new file mode 100644
index 000000000..bf59e7a98
--- /dev/null
+++ b/internal/host/restart_test.go
@@ -0,0 +1,706 @@
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "slices"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// restartEtag reads an app's restart-generation etag straight from the store,
+// so a test asserts on what the bucket actually reports rather than a value it
+// made up.
+func restartEtag(t *testing.T, ctx context.Context, s release.Store, acct, app string) string {
+ t.Helper()
+ _, etag, err := s.Get(ctx, release.RestartGenKey(acct, app))
+ if err != nil {
+ t.Fatalf("read restart generation %s/%s: %v", acct, app, err)
+ }
+ return etag
+}
+
+// askRestart is a member's `carlos restart` landing in the bucket: the one
+// write the whole feature turns on.
+func askRestart(t *testing.T, ctx context.Context, s release.Store, acct, app string) {
+ t.Helper()
+ if err := release.TouchRestartGen(ctx, s, acct, app); err != nil {
+ t.Fatalf("TouchRestartGen %s/%s: %v", acct, app, err)
+ }
+}
+
+// namedUnitRoute is the addr-backed, operator-named-unit shape issue #110
+// added and #113's second case is about: a console. Only the ROOT pass can
+// cycle it.
+func namedUnitRoute(host, acct, app, unit string) registry.Route {
+ rt := execRoute(host, acct, app)
+ rt.Socket = ""
+ rt.Addr = "127.0.0.1:9000"
+ rt.Unit = unit
+ return rt
+}
+
+// recordingRestart is the root pass's dispatch seam: every host it was asked
+// to cycle, in order, plus an optional per-host failure.
+type recordingRestart struct {
+ hosts []string
+ fail map[string]error
+}
+
+func (r *recordingRestart) fn(host string) error {
+ r.hosts = append(r.hosts, host)
+ if err, ok := r.fail[host]; ok {
+ return err
+ }
+ return nil
+}
+
+// ---- the agent's sweep -------------------------------------------------------
+
+func TestRestartGenPassFirstSightIsSilent(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven") // requested before this box ever looked
+ routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")}
+ cache := map[[2]string]string{}
+
+ moved, errs := restartGenPass(ctx, routes, s, cache)
+ if len(moved) != 0 {
+ t.Fatalf("first sight moved %v — a box that has just started must not cycle everything it serves", moved)
+ }
+ if len(errs) != 0 {
+ t.Fatalf("unexpected read errors: %v", errs)
+ }
+ if got, want := cache[[2]string{"bab", "eleven"}], restartEtag(t, ctx, s, "bab", "eleven"); got != want {
+ t.Fatalf("cache etag = %q, want the store's %q", got, want)
+ }
+}
+
+func TestRestartGenPassMovementAsksOnce(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven")
+ routes := []registry.Route{
+ execRoute("a.example.com", "bab", "eleven"),
+ execRoute("b.example.com", "bab", "eleven"), // same app, two routes
+ }
+ cache := map[[2]string]string{}
+ restartGenPass(ctx, routes, s, cache) // baseline
+
+ askRestart(t, ctx, s, "bab", "eleven")
+ moved, errs := restartGenPass(ctx, routes, s, cache)
+ if len(errs) != 0 {
+ t.Fatalf("unexpected read errors: %v", errs)
+ }
+ if len(moved) != 1 || moved[0] != [2]string{"bab", "eleven"} {
+ t.Fatalf("moved = %v, want exactly one app entry (one request, one ask)", moved)
+ }
+ if moved, _ := restartGenPass(ctx, routes, s, cache); len(moved) != 0 {
+ t.Fatalf("the pass after a request moved %v — a stale cache would ask forever", moved)
+ }
+}
+
+func TestRestartGenPassAbsentStaysSilentAndFirstRequestMoves(t *testing.T) {
+ ctx, s := genKit(t)
+ routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")}
+ cache := map[[2]string]string{}
+
+ // An app nobody has ever restarted: absent, legal, forever.
+ for i := range 2 {
+ moved, errs := restartGenPass(ctx, routes, s, cache)
+ if len(moved) != 0 {
+ t.Fatalf("pass %d: an app with no restart generation moved %v", i, moved)
+ }
+ if len(errs) != 0 {
+ t.Fatalf("pass %d: absent is not a read error: %v", i, errs)
+ }
+ }
+ if got := cache[[2]string{"bab", "eleven"}]; got != genAbsent {
+ t.Fatalf("cache = %q, want the known-absent sentinel %q", got, genAbsent)
+ }
+
+ // Its FIRST ever request: absent -> present is movement, or the very first
+ // `carlos restart` for an app would be the one that never works.
+ askRestart(t, ctx, s, "bab", "eleven")
+ moved, _ := restartGenPass(ctx, routes, s, cache)
+ if len(moved) != 1 {
+ t.Fatalf("absent -> present moved %v, want the app", moved)
+ }
+}
+
+func TestRestartGenPassReadErrorLeavesCacheAndKeepsGoing(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven")
+ askRestart(t, ctx, s, "bab", "titogo")
+ routes := []registry.Route{
+ execRoute("a.example.com", "bab", "eleven"),
+ execRoute("b.example.com", "bab", "titogo"),
+ }
+ cache := map[[2]string]string{}
+ restartGenPass(ctx, routes, s, cache)
+ before := cache[[2]string{"bab", "eleven"}]
+
+ askRestart(t, ctx, s, "bab", "titogo") // a real request on the app that CAN be read
+ blip := &failingKeyStore{Store: s, failKey: release.RestartGenKey("bab", "eleven"), err: errors.New("503 slow down")}
+ moved, errs := restartGenPass(ctx, routes, blip, cache)
+ if len(errs) != 1 {
+ t.Fatalf("read errors = %v, want exactly one", errs)
+ }
+ if len(moved) != 1 || moved[0] != [2]string{"bab", "titogo"} {
+ t.Fatalf("moved = %v, want only the readable app — one bad object must not stop the pass", moved)
+ }
+ if got := cache[[2]string{"bab", "eleven"}]; got != before {
+ t.Fatalf("cache for the unreadable app = %q, want it untouched at %q (a cleared entry is a bogus first sight)", got, before)
+ }
+}
+
+func TestRestartGenPassDropsVanishedApps(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven")
+ both := []registry.Route{
+ execRoute("a.example.com", "bab", "eleven"),
+ execRoute("b.example.com", "bab", "titogo"),
+ }
+ cache := map[[2]string]string{}
+ restartGenPass(ctx, both, s, cache)
+
+ restartGenPass(ctx, both[:1], s, cache) // titogo's route is gone
+ if _, ok := cache[[2]string{"bab", "titogo"}]; ok {
+ t.Fatal("a route that left the box kept its cache entry")
+ }
+ moved, _ := restartGenPass(ctx, both, s, cache) // and comes back
+ if len(moved) != 0 {
+ t.Fatalf("a re-added app moved %v, want a silent first sight", moved)
+ }
+}
+
+// TestRestartGenPassIgnoresNonAdoptableRoutes: the membership predicate is
+// release.Adoptable, the same one the adopt pass moves. A platform alias
+// shares its canonical route's process, and a mid-provision route must not be
+// disturbed — neither is an app this box should be reading a restart signal
+// for.
+func TestRestartGenPassIgnoresNonAdoptableRoutes(t *testing.T) {
+ ctx, s := genKit(t)
+ alias := execRoute("alias.oncarlos.com", "bab", "eleven")
+ alias.Platform = true
+ pending := execRoute("new.example.com", "bab", "titogo")
+ pending.Status = "creating"
+
+ cache := map[[2]string]string{}
+ restartGenPass(ctx, []registry.Route{alias, pending}, s, cache)
+ if len(cache) != 0 {
+ t.Fatalf("cache = %v, want empty (neither route is adoptable)", cache)
+ }
+}
+
+// ---- the agent's own children ------------------------------------------------
+
+func TestBounceExecRoutesCyclesOnlyWhatRootCannot(t *testing.T) {
+ moved := [][2]string{{"bab", "eleven"}}
+ routes := []registry.Route{
+ execRoute("exec.example.com", "bab", "eleven"), // the agent's child
+ unitRoute("unit.example.com", "bab", "eleven"), // carlos-app@ — root's
+ namedUnitRoute("console.example.com", "bab", "eleven", "console.service"), // #110 — root's
+ execRoute("other.example.com", "bab", "titogo"), // a different app
+ }
+ var bounced []string
+ n := bounceExecRoutes(routes, moved, func(h string) bool {
+ bounced = append(bounced, h)
+ return true
+ })
+ if n != 1 || !slices.Equal(bounced, []string{"exec.example.com"}) {
+ t.Fatalf("bounced %v (n=%d), want only the exec-backed route of the moved app", bounced, n)
+ }
+}
+
+// TestBounceExecRoutesReachesAWarmHibernator: a hibernate-flagged route is
+// included on purpose. Bounce cannot start anything, so a SLEEPING instance is
+// untouched — but a warm one is exactly the wedged instance `carlos restart`
+// exists for, and leaving it serving would be the feature failing at its own
+// headline case.
+func TestBounceExecRoutesReachesAWarmHibernator(t *testing.T) {
+ rt := execRoute("warm.example.com", "bab", "eleven")
+ rt.Hibernate = true
+ var bounced []string
+ // The activator's own contract: false when not awake, true when it stopped
+ // something. Here it is awake.
+ bounceExecRoutes([]registry.Route{rt}, [][2]string{{"bab", "eleven"}}, func(h string) bool {
+ bounced = append(bounced, h)
+ return true
+ })
+ if !slices.Equal(bounced, []string{"warm.example.com"}) {
+ t.Fatalf("bounced %v, want the warm hibernator", bounced)
+ }
+}
+
+func TestBounceExecRoutesWithoutAnActivator(t *testing.T) {
+ routes := []registry.Route{execRoute("exec.example.com", "bab", "eleven")}
+ if n := bounceExecRoutes(routes, [][2]string{{"bab", "eleven"}}, nil); n != 0 {
+ t.Fatalf("bounced %d with no activator armed, want 0 and no panic", n)
+ }
+}
+
+// ---- the agent's leg, and what makes it run ----------------------------------
+
+// TestRestartRequestLegBouncesAndRelays: one serial move's worth of work — the
+// agent's own children cycled in-process, and one marker touched for the units
+// only root can reach.
+func TestRestartRequestLegBouncesAndRelays(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ marker := filepath.Join(dd, restartRequestMarker)
+ routes := []registry.Route{
+ execRoute("exec.example.com", "bab", "eleven"),
+ unitRoute("unit.example.com", "bab", "eleven"),
+ }
+ var bounced []string
+ bounce := func(h string) bool { bounced = append(bounced, h); return true }
+ cache := map[[2]string]string{}
+
+ // Baseline (what restartSeed does at arming): silent, no marker.
+ askRestart(t, ctx, s, "bab", "eleven")
+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, cache); asked {
+ t.Fatal("the baselining leg asked for a root pass — first sight must be silent")
+ }
+ if _, err := os.Stat(marker); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("marker written on a first sight (err %v)", err)
+ }
+
+ askRestart(t, ctx, s, "bab", "eleven")
+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, cache); !asked {
+ t.Fatal("a moved restart generation must ask for a root pass")
+ }
+ if !slices.Equal(bounced, []string{"exec.example.com"}) {
+ t.Fatalf("bounced %v, want the exec child only (the unit route is root's)", bounced)
+ }
+ if _, err := os.Stat(marker); err != nil {
+ t.Fatalf("no marker after a request: %v", err)
+ }
+
+ // And a leg with nothing moved neither bounces nor re-asks.
+ bounced = nil
+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, cache); asked {
+ t.Fatal("an unmoved generation asked for a root pass")
+ }
+ if len(bounced) != 0 {
+ t.Fatalf("bounced %v with nothing moved", bounced)
+ }
+}
+
+// TestFastTickRunsTheRestartLegOnASerialMove pins the wiring the redesign rests
+// on: the restart check rides the SERIAL, which is what keeps an idle box at
+// one GET per tick. A boot seed must not run it (that would cycle processes for
+// bumps that happened while the box was down), an unchanged serial must not run
+// it, and a real move must — once, after the pass set.
+func TestFastTickRunsTheRestartLegOnASerialMove(t *testing.T) {
+ env := newFastFixture(t)
+ var passes, legs int
+ d := env.deps(func() { passes++ })
+ d.restartLeg = func(context.Context) {
+ legs++
+ if legs > passes {
+ t.Errorf("the restart leg ran before/without the pass set (legs=%d passes=%d)", legs, passes)
+ }
+ }
+ now := time.Unix(1754640000, 0)
+ ctx := context.Background()
+
+ if err := release.BumpSerial(ctx, env.store, now); err != nil {
+ t.Fatalf("BumpSerial: %v", err)
+ }
+ fastTick(ctx, d, now) // first sight of a present serial: seed only
+ if legs != 0 {
+ t.Fatalf("boot seed ran the restart leg %d times, want 0", legs)
+ }
+ fastTick(ctx, d, now.Add(10*time.Second)) // unchanged
+ if legs != 0 {
+ t.Fatalf("unchanged serial ran the restart leg %d times, want 0", legs)
+ }
+ if err := release.BumpSerial(ctx, env.store, now); err != nil {
+ t.Fatalf("BumpSerial: %v", err)
+ }
+ fastTick(ctx, d, now.Add(20*time.Second))
+ if legs != 1 {
+ t.Fatalf("a serial move ran the restart leg %d times, want 1", legs)
+ }
+}
+
+// ---- the root pass -----------------------------------------------------------
+
+func marksOf(t *testing.T, path string) *restartMarks {
+ t.Helper()
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("read marks: %v", err)
+ }
+ m := &restartMarks{}
+ if err := json.Unmarshal(raw, m); err != nil {
+ t.Fatalf("decode marks %s: %v", raw, err)
+ }
+ return m
+}
+
+// TestRestartPassSeedsWithoutReplaying is the expensive lesson this repo has
+// already paid for once: a box that has never recorded a baseline records one
+// and restarts NOTHING. Every app on a freshly upgraded box has a restart
+// generation the moment anyone has ever used the verb, and cycling them all on
+// the first pass would take the box out.
+func TestRestartPassSeedsWithoutReplaying(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven")
+ askRestart(t, ctx, s, "bab", "titogo")
+ routes := []registry.Route{
+ unitRoute("a.example.com", "bab", "eleven"),
+ unitRoute("b.example.com", "bab", "titogo"),
+ }
+ path := filepath.Join(t.TempDir(), "restart-marks.json")
+ rec := &recordingRestart{}
+
+ restartPass(ctx, routes, s, path, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("the seeding pass cycled %v, want nothing", rec.hosts)
+ }
+ m := marksOf(t, path)
+ if len(m.Apps) != 2 {
+ t.Fatalf("marks = %v, want both apps baselined", m.Apps)
+ }
+ // And the very next pass is a quiet one, not a replay.
+ restartPass(ctx, routes, s, path, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("the pass after seeding cycled %v, want nothing", rec.hosts)
+ }
+}
+
+// TestRestartPassCyclesOnMovement is the feature working: baseline, request,
+// dispatch — for every non-hibernating route of THAT app and no other.
+func TestRestartPassCyclesOnMovement(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven")
+ askRestart(t, ctx, s, "bab", "titogo")
+ sleeping := unitRoute("sleep.example.com", "bab", "eleven")
+ sleeping.Hibernate = true
+ routes := []registry.Route{
+ unitRoute("a.example.com", "bab", "eleven"),
+ namedUnitRoute("console.example.com", "bab", "eleven", "console.service"),
+ sleeping,
+ unitRoute("z.example.com", "bab", "titogo"), // a bystander app
+ }
+ path := filepath.Join(t.TempDir(), "restart-marks.json")
+ rec := &recordingRestart{}
+ restartPass(ctx, routes, s, path, rec.fn) // baseline
+
+ askRestart(t, ctx, s, "bab", "eleven")
+ restartPass(ctx, routes, s, path, rec.fn)
+
+ slices.Sort(rec.hosts)
+ want := []string{"a.example.com", "console.example.com"}
+ if !slices.Equal(rec.hosts, want) {
+ t.Fatalf("cycled %v, want %v — the hibernating route must be left asleep and another app's route untouched",
+ rec.hosts, want)
+ }
+ if got, want := marksOf(t, path).Apps[restartMarkKey("bab", "eleven")], restartEtag(t, ctx, s, "bab", "eleven"); got != want {
+ t.Fatalf("mark = %q, want the etag just acted on %q", got, want)
+ }
+ // Idempotent: the same request is not served twice.
+ rec.hosts = nil
+ restartPass(ctx, routes, s, path, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("a second pass on the same generation cycled %v, want nothing", rec.hosts)
+ }
+}
+
+// TestRestartPassActsOnAnAppsFirstEverRequest: an app with no object yet is
+// recorded as ABSENT, and the object's later appearance is movement away from
+// that recorded value. Both halves matter — without the recorded absence, the
+// first `carlos restart` for every app is either dropped (if "no mark" seeds)
+// or joined by a spurious cycle of every other app in the same state (if "no
+// mark" dispatches).
+func TestRestartPassActsOnAnAppsFirstEverRequest(t *testing.T) {
+ ctx, s := genKit(t)
+ routes := []registry.Route{unitRoute("a.example.com", "bab", "eleven")}
+ path := filepath.Join(t.TempDir(), "restart-marks.json")
+ rec := &recordingRestart{}
+
+ restartPass(ctx, routes, s, path, rec.fn) // baseline: the app has no object at all
+ if got := marksOf(t, path).Apps[restartMarkKey("bab", "eleven")]; got != restartAbsent {
+ t.Fatalf("mark = %q, want the recorded absent sentinel", got)
+ }
+
+ askRestart(t, ctx, s, "bab", "eleven")
+ restartPass(ctx, routes, s, path, rec.fn)
+ if !slices.Equal(rec.hosts, []string{"a.example.com"}) {
+ t.Fatalf("cycled %v, want the app's route on its first ever request", rec.hosts)
+ }
+}
+
+// TestRestartPassSeedsAnAppItHasNeverSeen is the storm fix (review, Critical).
+// A route added to this box for an app that was restarted somewhere else weeks
+// ago must be RECORDED, not cycled: this box owes nothing to a request it never
+// saw, and reading "no mark" as movement is what turned a registry that landed
+// after the marks file into a box that try-restarted its entire route table in
+// one pass.
+func TestRestartPassSeedsAnAppItHasNeverSeen(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven")
+ askRestart(t, ctx, s, "bab", "titogo") // both restarted long ago, elsewhere
+ first := []registry.Route{unitRoute("a.example.com", "bab", "eleven")}
+ path := filepath.Join(t.TempDir(), "restart-marks.json")
+ rec := &recordingRestart{}
+ restartPass(ctx, first, s, path, rec.fn) // this box knows about eleven only
+
+ // titogo's route arrives now — a fresh instance, mid-provision until this
+ // moment. Its app's object has existed for weeks.
+ both := append(first, unitRoute("b.example.com", "bab", "titogo"))
+ restartPass(ctx, both, s, path, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("a newly-seen app cycled %v, want nothing — this box never saw that request", rec.hosts)
+ }
+ if got, want := marksOf(t, path).Apps[restartMarkKey("bab", "titogo")], restartEtag(t, ctx, s, "bab", "titogo"); got != want {
+ t.Fatalf("mark = %q, want the seeded etag %q", got, want)
+ }
+ // And it is live from here: the NEXT request cycles it.
+ askRestart(t, ctx, s, "bab", "titogo")
+ restartPass(ctx, both, s, path, rec.fn)
+ if !slices.Equal(rec.hosts, []string{"b.example.com"}) {
+ t.Fatalf("cycled %v, want the newly-seeded app's route on the next real request", rec.hosts)
+ }
+}
+
+// TestRestartPassWithNoRoutesWritesNothing: a box whose registry has not landed
+// yet (fresh machine, /data restoring from litestream) must not be able to
+// record "I know about no apps" — that file is what makes every route it later
+// gains look like movement.
+func TestRestartPassWithNoRoutesWritesNothing(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven")
+ path := filepath.Join(t.TempDir(), "restart-marks.json")
+ rec := &recordingRestart{}
+
+ restartPass(ctx, nil, s, path, rec.fn)
+ if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("a pass with zero adoptable routes wrote %s (err %v) — it knows nothing and may claim nothing", path, err)
+ }
+ // The registry lands; the app is seeded, not cycled.
+ routes := []registry.Route{unitRoute("a.example.com", "bab", "eleven")}
+ restartPass(ctx, routes, s, path, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("cycled %v after the registry landed, want a silent baseline", rec.hosts)
+ }
+}
+
+// TestRestartPassDefersABaselineWithReadErrors: a baseline that omits the apps
+// a transient 503 hid is a baseline that turns those apps into movement on the
+// next pass. Nothing is written until a pass can read every app.
+func TestRestartPassDefersABaselineWithReadErrors(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven")
+ askRestart(t, ctx, s, "bab", "titogo")
+ routes := []registry.Route{
+ unitRoute("a.example.com", "bab", "eleven"),
+ unitRoute("b.example.com", "bab", "titogo"),
+ }
+ path := filepath.Join(t.TempDir(), "restart-marks.json")
+ rec := &recordingRestart{}
+
+ blip := &failingKeyStore{Store: s, failKey: release.RestartGenKey("bab", "eleven"), err: errors.New("503 slow down")}
+ restartPass(ctx, routes, blip, path, rec.fn)
+ if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("a baselining pass with a read error wrote %s (err %v)", path, err)
+ }
+
+ // A clean pass baselines both, and cycles neither.
+ restartPass(ctx, routes, s, path, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("the clean baselining pass cycled %v, want nothing", rec.hosts)
+ }
+ if m := marksOf(t, path); len(m.Apps) != 2 {
+ t.Fatalf("marks = %v, want both apps", m.Apps)
+ }
+}
+
+// TestRestartPassRefusesWithoutALock: no lock means either a concurrent root
+// pass or a /data this pass cannot write. Dispatching under the first
+// double-restarts one request; under the second it restarts hourly forever,
+// since the mark can never advance. Neither is allowed to happen.
+func TestRestartPassRefusesWithoutALock(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven")
+ dir := t.TempDir()
+ path := filepath.Join(dir, "restart-marks.json")
+ routes := []registry.Route{unitRoute("a.example.com", "bab", "eleven")}
+ rec := &recordingRestart{}
+ restartPass(ctx, routes, s, path, rec.fn) // baseline, with a working lock
+
+ // A directory where the lock file needs to be: opening it fails (EISDIR),
+ // exactly as opening it would fail on a read-only or full /data.
+ if err := os.Remove(path + ".lock"); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Mkdir(path+".lock", 0o755); err != nil {
+ t.Fatal(err)
+ }
+ askRestart(t, ctx, s, "bab", "eleven")
+ restartPass(ctx, routes, s, path, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("cycled %v with no lock — an unrecordable restart repeats forever", rec.hosts)
+ }
+}
+
+// TestRestartPassSerializesConcurrentPasses: three root units can run this at
+// once (carlos-restart, carlos-stage, carlos-adopt). Under the lock, one
+// request produces ONE dispatch however many passes race for it.
+func TestRestartPassSerializesConcurrentPasses(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven")
+ path := filepath.Join(t.TempDir(), "restart-marks.json")
+ routes := []registry.Route{unitRoute("a.example.com", "bab", "eleven")}
+ var mu sync.Mutex
+ var hosts []string
+ dispatch := func(h string) error {
+ mu.Lock()
+ defer mu.Unlock()
+ hosts = append(hosts, h)
+ return nil
+ }
+ restartPass(ctx, routes, s, path, dispatch) // baseline
+
+ askRestart(t, ctx, s, "bab", "eleven")
+ var wg sync.WaitGroup
+ for range 3 {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ restartPass(ctx, routes, s, path, dispatch)
+ }()
+ }
+ wg.Wait()
+ if len(hosts) != 1 {
+ t.Fatalf("three concurrent passes dispatched %v, want exactly one — the flock is what makes a request singular", hosts)
+ }
+}
+
+// TestRestartPassAdvancesTheMarkOnAFailedDispatch: a restart request is a
+// point-in-time event, not a desired state. A named unit that will not come
+// back must be reported, not retried on every pass until the end of time.
+func TestRestartPassAdvancesTheMarkOnAFailedDispatch(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven")
+ routes := []registry.Route{namedUnitRoute("console.example.com", "bab", "eleven", "wrong.service")}
+ path := filepath.Join(t.TempDir(), "restart-marks.json")
+ rec := &recordingRestart{fail: map[string]error{"console.example.com": errors.New("Unit wrong.service not loaded")}}
+ restartPass(ctx, routes, s, path, rec.fn) // baseline
+
+ askRestart(t, ctx, s, "bab", "eleven")
+ restartPass(ctx, routes, s, path, rec.fn)
+ if len(rec.hosts) != 1 {
+ t.Fatalf("dispatched %v, want one attempt", rec.hosts)
+ }
+ rec.hosts = nil
+ restartPass(ctx, routes, s, path, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("a failed dispatch was retried (%v) — the request must not become a standing order", rec.hosts)
+ }
+}
+
+// TestRestartPassUnreadableMarksReBaseline: a wiped or corrupt marks file
+// fails toward NOT restarting. The cost is one dropped request, logged and
+// re-runnable; the alternative is a box that cycles everything it serves the
+// moment a state file goes missing.
+func TestRestartPassUnreadableMarksReBaseline(t *testing.T) {
+ ctx, s := genKit(t)
+ askRestart(t, ctx, s, "bab", "eleven")
+ routes := []registry.Route{unitRoute("a.example.com", "bab", "eleven")}
+ path := filepath.Join(t.TempDir(), "restart-marks.json")
+ if err := os.WriteFile(path, []byte("{not json"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ rec := &recordingRestart{}
+
+ restartPass(ctx, routes, s, path, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("a corrupt marks file cycled %v, want nothing", rec.hosts)
+ }
+ if got, want := marksOf(t, path).Apps[restartMarkKey("bab", "eleven")], restartEtag(t, ctx, s, "bab", "eleven"); got != want {
+ t.Fatalf("re-baselined mark = %q, want %q", got, want)
+ }
+}
+
+// ---- the handover between the two halves -------------------------------------
+
+// TestSeedRestartMarksIsCreateOnly: the agent writes the baseline as it arms,
+// so a request made before the root pass has ever run is not silently dropped
+// — but it must never overwrite a mark the root pass has already advanced, or
+// a served request would be served again. Absent sightings ARE recorded: that
+// sentinel is what makes an app's first ever request movement.
+func TestSeedRestartMarksIsCreateOnly(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "restart-marks.json")
+ cache := map[[2]string]string{
+ {"bab", "eleven"}: "etag-1",
+ {"bab", "titogo"}: genAbsent, // seen, and it has no object
+ }
+ if err := seedRestartMarks(path, cache); err != nil {
+ t.Fatalf("seed: %v", err)
+ }
+ m := marksOf(t, path)
+ if m.Apps[restartMarkKey("bab", "eleven")] != "etag-1" ||
+ m.Apps[restartMarkKey("bab", "titogo")] != restartAbsent {
+ t.Fatalf("seeded marks = %v, want the etag and the recorded absence", m.Apps)
+ }
+
+ cache[[2]string{"bab", "eleven"}] = "etag-2"
+ if err := seedRestartMarks(path, cache); err != nil {
+ t.Fatalf("second seed: %v", err)
+ }
+ if got := marksOf(t, path).Apps[restartMarkKey("bab", "eleven")]; got != "etag-1" {
+ t.Fatalf("mark = %q after a second seed, want the original %q — seeding must never overwrite", got, "etag-1")
+ }
+}
+
+// TestSeedRestartMarksRefusesAnEmptyBaseline: a sweep that saw nothing is most
+// often a box whose registry has not landed yet. A file claiming "this box
+// knows about no apps" is exactly what makes the root pass treat every route
+// the box later gains as movement.
+func TestSeedRestartMarksRefusesAnEmptyBaseline(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "restart-marks.json")
+ if err := seedRestartMarks(path, map[[2]string]string{}); err != nil {
+ t.Fatalf("seed: %v", err)
+ }
+ if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("an empty sweep wrote %s (err %v), want no file at all", path, err)
+ }
+}
+
+// TestSeededBaselineIsWhatTheRootPassTrusts closes the handover: the agent
+// seeds, a member then asks, and the root pass — running for the FIRST time,
+// with no memory of its own — still cycles. Without the seed that pass would
+// baseline instead and the request would vanish having been answered
+// "requested".
+func TestSeededBaselineIsWhatTheRootPassTrusts(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ routes := []registry.Route{unitRoute("a.example.com", "bab", "eleven")}
+ askRestart(t, ctx, s, "bab", "eleven") // an older request, already served
+
+ // The agent's first tick: baseline in memory, baseline on disk.
+ cache := map[[2]string]string{}
+ restartGenPass(ctx, routes, s, cache)
+ if err := seedRestartMarks(restartMarksPath(dd), cache); err != nil {
+ t.Fatalf("seed: %v", err)
+ }
+
+ askRestart(t, ctx, s, "bab", "eleven") // the member types `carlos restart`
+ moved, _ := restartGenPass(ctx, routes, s, cache)
+ if len(moved) != 1 {
+ t.Fatalf("the agent missed the request: moved = %v", moved)
+ }
+ rec := &recordingRestart{}
+ restartPass(ctx, routes, s, restartMarksPath(dd), rec.fn)
+ if !slices.Equal(rec.hosts, []string{"a.example.com"}) {
+ t.Fatalf("the first root pass cycled %v, want the requested route", rec.hosts)
+ }
+}
diff --git a/internal/host/units/carlos-restart.path b/internal/host/units/carlos-restart.path
new file mode 100644
index 000000000..f5e20875d
--- /dev/null
+++ b/internal/host/units/carlos-restart.path
@@ -0,0 +1,27 @@
+# Member-requested process restarts (issue #113) — carlos-adopt.path's twin
+# for the third half the sandboxed agent cannot do itself. The agent's
+# serial poll sees the console's bump within ~2s, compares the app's restart
+# generation, and bounces its OWN exec children immediately — but it cannot
+# restart a systemd unit: not a carlos-app@ tenant, and not the bespoke unit an
+# addr-backed route names (issue #110's consoles). Touching
+# /data/restart-requested relays that to a root pass.
+#
+# It starts carlos-restart.service — `carlos adopt -restart-only` —
+# deliberately NOT carlos-adopt.service or carlos-config-sync.service. A
+# restart request changes no version and writes no config, so a full pass would
+# stage releases and nag about canaries on a per-request trigger, and the
+# config pass would find no changed env file and cycle nothing at all. Same
+# split, and the same reason, as carlos-stage.path having its own service.
+#
+# /data is hardcoded like every path in these unit files — a box with a
+# nonstandard data dir simply never fast-triggers and keeps the hourly floor
+# (carlos-adopt.timer runs the same comparison).
+[Unit]
+Description=CARLOS instance restart on agent request (carlos restart)
+
+[Path]
+PathChanged=/data/restart-requested
+Unit=carlos-restart.service
+
+[Install]
+WantedBy=multi-user.target
diff --git a/internal/host/units/carlos-restart.service b/internal/host/units/carlos-restart.service
new file mode 100644
index 000000000..fe55e1440
--- /dev/null
+++ b/internal/host/units/carlos-restart.service
@@ -0,0 +1,53 @@
+# One RESTART-REQUEST pass (`carlos restart`, issue #113): cycle the units of
+# every app whose restart generation has moved since this box last acted, and
+# nothing else. Started only by carlos-restart.path, which fires when the agent
+# touches /data/restart-requested; it has no [Install] section because nothing
+# else should ever want it.
+#
+# Deliberately NOT carlos-adopt.service: that unit also runs release.Pass, the
+# capability converge and the canary-TTL nag, so pointing a per-request trigger
+# at it would let one tenant's `carlos restart` cut over ANOTHER app's pending
+# release within seconds, and would nag on every request instead of hourly —
+# the same split carlos-config-sync.service and carlos-stage.service already
+# make. Same root + hardening shape as those two, which is what `systemctl
+# try-restart` requires.
+#
+# It needs neither CARLOS_RELEASE_PUBKEY nor CARLOS_SECRETS_KEY: a restart
+# verifies no signature and renders no config. The EnvironmentFile is still
+# read for CARLOS_DEPLOYMENT_BUCKET/_DIR, which is where the request itself is.
+[Unit]
+Description=carlos instance restart (on agent request)
+After=network-online.target
+Wants=network-online.target
+# A path unit re-triggers as often as members ask; the default start rate limit
+# would latch this unit into start-limit-hit after a burst and then ignore every
+# later trigger until an operator reset it. ([Unit], not [Service] — systemd
+# moved these two keys in v230.)
+StartLimitIntervalSec=0
+
+[Service]
+Type=oneshot
+# CARLOS_DEPLOYMENT_BUCKET/_DIR lives here.
+EnvironmentFile=-/etc/carlos/host.env
+ExecStart=/opt/carlos/carlos adopt -restart-only
+
+# --- hardening ---
+# Runs as ROOT deliberately, exactly like carlos-adopt.service: this pass
+# restarts carlos-app@ tenants and operator-named units via systemctl.
+# NoNewPrivileges and the syscall filter are omitted for the systemctl child;
+# the rest applies.
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectClock=yes
+ProtectHostname=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+RestrictNamespaces=yes
+RestrictRealtime=yes
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+MemoryMax=256M
+TasksMax=64
diff --git a/internal/release/restartgen.go b/internal/release/restartgen.go
new file mode 100644
index 000000000..6ca18524f
--- /dev/null
+++ b/internal/release/restartgen.go
@@ -0,0 +1,107 @@
+// The restart signal: one tiny object per app that a restart REQUEST touches,
+// so a box can learn "cycle this app's processes" from a single cheap etag —
+// the pull-based answer to issue #113 (a member must be able to cycle a wedged
+// instance with no version and no config to push, and with no box access at
+// all). Deliberately the same shape as appconfig's config-gen object, because
+// it is the same mechanism one step sideways: a generation object nobody
+// signs, watched by etag, whose movement means "do the thing", never "trust
+// this content".
+//
+// Bucket layout addition:
+//
+// apps/<acct>/<app>/restart-gen.json CAS-guarded, schema-1 generation object
+package release
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "time"
+)
+
+// RestartGeneration is the object `carlos restart` touches —
+// apps/<acct>/<app>/restart-gen.json. Change detection is by ETAG, never by
+// content: the fields exist so a human debugging "why did this bounce?" can
+// see when the last request went out, and so each touch's bytes are distinct
+// (DirStore's etag is content-derived, so a byte-identical rewrite would not
+// move it, and the movement IS the signal).
+//
+// It carries no authority whatsoever, which is what makes an unsigned object
+// safe here: the worst a forged touch can do is restart processes that were
+// already running the version their own signed pointer names. Nothing about
+// what runs is decided by this file.
+//
+// ABSENT IS LEGAL FOREVER. An app that has never been restarted this way has
+// no object and never will until someone asks; boxes read the absence, record
+// it, and go on. Old boxes that predate this feature ignore the key entirely
+// — the wire shape is purely additive, and nothing that parses today's app
+// prefix parses this.
+//
+// It is a SIBLING object rather than a field on anything existing for
+// appconfig.Generation's reason: a field would make every watcher re-read and
+// re-parse a whole manifest to learn one bit, and would race the very write
+// it signals.
+type RestartGeneration struct {
+ Schema int `json:"schema"` // 1
+ TouchedAt string `json:"touched_at"`
+ Nonce string `json:"nonce"`
+}
+
+// RestartGenKey is the bucket key of app's restart-generation object, derived
+// from AppPrefix like every other per-app key: everything an app owns sits
+// under one account-scoped prefix, permanently.
+func RestartGenKey(acct, app string) string { return AppPrefix(acct, app) + "restart-gen.json" }
+
+// TouchRestartGen bumps app's restart generation — the request every box
+// serving that app converges on within seconds.
+//
+// A CAS conflict is SUCCESS, not an error: it means another writer touched the
+// object between our read and our write, so the signal this function exists to
+// fire has already fired. Retrying would only produce a second, redundant
+// request — and a second restart of processes that are cycling right now is
+// the one outcome a restart verb must not manufacture out of a race.
+//
+// os.ErrNotExist on the read means first touch, so the write is create-only
+// (etag ""), which is what makes the conflict carve-out safe: two first
+// touches race, one wins, both callers are correct — both asked for a restart,
+// and exactly one restart is what the box will do.
+func TouchRestartGen(ctx context.Context, s Store, acct, app string) error {
+ if !ValidAccount(acct) {
+ return fmt.Errorf("restart: invalid account %q", acct)
+ }
+ if !ValidApp(app) {
+ return fmt.Errorf("restart: invalid app %q", app)
+ }
+ key := RestartGenKey(acct, app)
+ _, etag, err := s.Get(ctx, key)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ if errors.Is(err, os.ErrNotExist) {
+ etag = ""
+ }
+ nonce := make([]byte, 16)
+ if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+ return err
+ }
+ raw, err := json.Marshal(RestartGeneration{
+ Schema: 1,
+ TouchedAt: time.Now().UTC().Format(time.RFC3339),
+ Nonce: hex.EncodeToString(nonce),
+ })
+ if err != nil {
+ return err
+ }
+ if _, err := s.Put(ctx, key, raw, etag); err != nil {
+ if errors.Is(err, ErrCASConflict) {
+ return nil
+ }
+ return err
+ }
+ return nil
+}
diff --git a/internal/release/restartgen_test.go b/internal/release/restartgen_test.go
new file mode 100644
index 000000000..afdf59581
--- /dev/null
+++ b/internal/release/restartgen_test.go
@@ -0,0 +1,135 @@
+package release
+
+import (
+ "context"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "os"
+ "testing"
+ "time"
+)
+
+// restartEtag reads the app's restart-generation etag, "" for absent — the
+// exact observation a box's sweep makes, so a test asserting on it asserts on
+// the signal itself rather than on this file's internals.
+func restartEtag(t *testing.T, s Store, acct, app string) string {
+ t.Helper()
+ _, etag, err := s.Get(context.Background(), RestartGenKey(acct, app))
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return ""
+ }
+ t.Fatalf("read restart generation: %v", err)
+ }
+ return etag
+}
+
+func TestTouchRestartGenCreatesAndMoves(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ if err := TouchRestartGen(ctx, s, "bab", "keymail"); err != nil {
+ t.Fatalf("first TouchRestartGen: %v", err)
+ }
+ raw, first, err := s.Get(ctx, RestartGenKey("bab", "keymail"))
+ if err != nil {
+ t.Fatalf("Get after first touch: %v", err)
+ }
+ var g RestartGeneration
+ if err := json.Unmarshal(raw, &g); err != nil {
+ t.Fatalf("decode restart generation: %v", err)
+ }
+ if g.Schema != 1 {
+ t.Fatalf("Schema = %d, want 1", g.Schema)
+ }
+ if _, err := time.Parse(time.RFC3339, g.TouchedAt); err != nil {
+ t.Fatalf("TouchedAt %q is not RFC3339: %v", g.TouchedAt, err)
+ }
+ if nonce, err := hex.DecodeString(g.Nonce); err != nil || len(nonce) == 0 {
+ t.Fatalf("Nonce = %q, want nonempty hex (decode err %v)", g.Nonce, err)
+ }
+ // The etag is the whole point: a second request must be a DIFFERENT
+ // object, or the sweep watching it sees nothing and nothing restarts.
+ if err := TouchRestartGen(ctx, s, "bab", "keymail"); err != nil {
+ t.Fatalf("second TouchRestartGen: %v", err)
+ }
+ if second := restartEtag(t, s, "bab", "keymail"); second == first {
+ t.Fatalf("etag %q unchanged after a second request — the restart signal never fires", second)
+ }
+}
+
+// racingRestartPutStore lands a different writer's value at key just before
+// forwarding the caller's Put, which then fails CAS against the etag the
+// caller read — two people (or a person and a retrying CI job) asking for a
+// restart at the same moment.
+type racingRestartPutStore struct {
+ Store
+ key string
+ done bool
+}
+
+func (r *racingRestartPutStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
+ if key == r.key && !r.done {
+ r.done = true
+ other := []byte(`{"schema":1,"touched_at":"2026-01-01T00:00:00Z","nonce":"00112233445566778899aabbccddeeff"}`)
+ if _, err := r.Store.Put(ctx, key, other, ifMatch); err != nil {
+ return "", err
+ }
+ }
+ return r.Store.Put(ctx, key, body, ifMatch)
+}
+
+func TestTouchRestartGenCASConflictIsSuccess(t *testing.T) {
+ inner := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ if err := TouchRestartGen(ctx, inner, "bab", "keymail"); err != nil {
+ t.Fatalf("seed TouchRestartGen: %v", err)
+ }
+ racing := &racingRestartPutStore{Store: inner, key: RestartGenKey("bab", "keymail")}
+ if err := TouchRestartGen(ctx, racing, "bab", "keymail"); err != nil {
+ t.Fatalf("TouchRestartGen on a lost CAS race: %v, want nil (the request already fired)", err)
+ }
+ if !racing.done {
+ t.Fatal("the racing writer never ran — this test proved nothing")
+ }
+}
+
+// failingRestartPutStore fails every Put with a fixed non-CAS error.
+type failingRestartPutStore struct {
+ Store
+ err error
+}
+
+func (f *failingRestartPutStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
+ return "", f.err
+}
+
+// TestTouchRestartGenSurfacesRealPutErrors is the other half of the CAS
+// carve-out: ONLY ErrCASConflict is success. Any other store failure means
+// nobody asked for anything, and a caller told "requested" would stand there
+// waiting for a restart that is never coming.
+func TestTouchRestartGenSurfacesRealPutErrors(t *testing.T) {
+ s := &failingRestartPutStore{Store: NewDirStore(t.TempDir()), err: errors.New("bucket is on fire")}
+ if err := TouchRestartGen(context.Background(), s, "bab", "keymail"); err == nil {
+ t.Fatal("TouchRestartGen swallowed a non-CAS Put error, want it surfaced")
+ }
+}
+
+func TestTouchRestartGenRefusesInvalidNames(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ if err := TouchRestartGen(ctx, s, "../etc", "keymail"); err == nil {
+ t.Fatal("TouchRestartGen accepted an invalid account")
+ }
+ if err := TouchRestartGen(ctx, s, "bab", "../etc"); err == nil {
+ t.Fatal("TouchRestartGen accepted an invalid app")
+ }
+}
+
+// TestRestartGenKeyIsAnAppSibling pins the key's shape: under the app's own
+// account-scoped prefix, beside config-gen.json rather than inside anything.
+func TestRestartGenKeyIsAnAppSibling(t *testing.T) {
+ if got, want := RestartGenKey("bab", "keymail"), "apps/bab/keymail/restart-gen.json"; got != want {
+ t.Fatalf("RestartGenKey = %q, want %q", got, want)
+ }
+}
diff --git a/internal/status/contract_test.go b/internal/status/contract_test.go
index 3c485445b..075569d76 100644
--- a/internal/status/contract_test.go
+++ b/internal/status/contract_test.go
@@ -39,6 +39,13 @@ var allowedBoxKeys = map[string][]string{
// on the hourly floor. Same publication argument — a platform-own unit's
// liveness on an opaquely-labelled box, no tenant in it anywhere.
//
+ // "restart_path_running" is the issue #113 sibling of both: whether this
+ // box's restart trigger unit (carlos-restart.path) is armed, i.e. whether a
+ // member's `carlos restart` reaches its units in seconds or waits out the
+ // hourly floor. Identical publication argument — one platform-own unit's
+ // liveness on an opaquely-labelled box, and no tenant anywhere in it (it
+ // says nothing about WHICH app was restarted, or that any was).
+ //
// "metrics" is the one key in this allowlist that is EXEMPT from the
// band-word-not-a-number rule every other Box field follows (Task 5,
// internal/status/snapshot.go's package doc): it carries raw byte
@@ -56,7 +63,7 @@ var allowedBoxKeys = map[string][]string{
// safe for exactly the same bucket-private reason: BuildFleet's
// field-by-field copy never names "metrics" at all, so no hostname here
// can reach FleetBox or fleet.json regardless of what this map holds.
- "": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "config_path_running", "expired_canaries", "metrics"},
+ "": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "config_path_running", "restart_path_running", "expired_canaries", "metrics"},
"routes": {"static", "resident", "hibernating", "awake", "asleep"},
"replication": {"healthy", "lag_seconds"},
"metrics": {"mem_total_bytes", "mem_avail_bytes", "load1", "disk_free_bytes", "release", "tunnel", "route_versions"},
@@ -67,7 +74,7 @@ var allowedBoxKeys = map[string][]string{
// allowedBoxKeys: every new field is a deliberate publication decision.
var allowedFleetKeys = map[string][]string{
"": {"generated_at", "boxes", "instances", "capacity", "apps", "apps_unlisted", "services", "replication", "adoption_running", "expired_canaries"},
- "boxes[]": {"label", "region", "load", "stale", "routes", "adoption_running", "config_path_running", "expired_canaries"},
+ "boxes[]": {"label", "region", "load", "stale", "routes", "adoption_running", "config_path_running", "restart_path_running", "expired_canaries"},
"boxes[].routes": {"static", "resident", "hibernating", "awake", "asleep"},
"instances": {"total", "static", "resident", "hibernating", "awake", "asleep"},
"capacity": {"level"},
diff --git a/internal/status/dryrun.go b/internal/status/dryrun.go
index 4143f75a0..aec3da27b 100644
--- a/internal/status/dryrun.go
+++ b/internal/status/dryrun.go
@@ -141,6 +141,7 @@ func dryRunFixtures() ([]registry.Route, Probes) {
lag := int64(120)
adopting := true // a known reading, so adoption_running publishes non-null and gets scanned
configPath := true // ditto for config_path_running (issue #44)
+ restartPath := true // ditto for restart_path_running (issue #113)
expiredCanaries := 2 // ditto for expired_canaries — a known count, not the unexercised nil path
// Task 5's raw Metrics fixtures: distinctive values (see
@@ -151,14 +152,15 @@ func dryRunFixtures() ([]registry.Route, Probes) {
load1 := 42.42
probes := Probes{
- MachineID: "ip-10-0-0-7",
- Region: "us-east-1",
- MemFrac: &memFrac,
- DiskFrac: &diskFrac,
- LagSecs: &lag,
- AdoptRunning: &adopting,
- ConfigPathRunning: &configPath,
- ExpiredCanaries: &expiredCanaries,
+ MachineID: "ip-10-0-0-7",
+ Region: "us-east-1",
+ MemFrac: &memFrac,
+ DiskFrac: &diskFrac,
+ LagSecs: &lag,
+ AdoptRunning: &adopting,
+ ConfigPathRunning: &configPath,
+ RestartPathRunning: &restartPath,
+ ExpiredCanaries: &expiredCanaries,
// Every hibernating route reports "listening" so Awake/Asleep publish
// non-null — an unexercised null field is a field DryRun can't scan.
Listening: func(string) *bool { v := true; return &v },
diff --git a/internal/status/fleet.go b/internal/status/fleet.go
index eaf12a48a..463f4b656 100644
--- a/internal/status/fleet.go
+++ b/internal/status/fleet.go
@@ -50,6 +50,11 @@ type FleetBox struct {
// page shows a tile for them, and this one is a per-box operational
// fact, not a headline about the fleet.
ConfigPathRunning *bool `json:"config_path_running"`
+ // RestartPathRunning appears per-box for exactly the reason
+ // ConfigPathRunning does, and is not rolled up for the same reason: it is a
+ // per-box operational fact ("this box's restart trigger is not watching"),
+ // not a headline about the fleet.
+ RestartPathRunning *bool `json:"restart_path_running"`
// ExpiredCanaries appears per-box for the same reason AdoptionRunning
// does: the fleet-wide sum says something is expired somewhere, this
// says on which box, without naming which route. Published in the FEED
@@ -165,14 +170,15 @@ func BuildFleet(ctx context.Context, s release.Store, routes []registry.Route, n
for i, b := range boxes {
stale[i] = BoxStale(b, now, period)
f.Boxes = append(f.Boxes, FleetBox{
- Label: b.Label,
- Region: b.Region,
- Load: b.Load,
- Stale: stale[i],
- Routes: b.Routes,
- AdoptionRunning: b.AdoptionRunning,
- ConfigPathRunning: b.ConfigPathRunning,
- ExpiredCanaries: b.ExpiredCanaries,
+ Label: b.Label,
+ Region: b.Region,
+ Load: b.Load,
+ Stale: stale[i],
+ Routes: b.Routes,
+ AdoptionRunning: b.AdoptionRunning,
+ ConfigPathRunning: b.ConfigPathRunning,
+ RestartPathRunning: b.RestartPathRunning,
+ ExpiredCanaries: b.ExpiredCanaries,
})
}
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go
index 5fb6f128e..39b18fc08 100644
--- a/internal/status/snapshot.go
+++ b/internal/status/snapshot.go
@@ -59,6 +59,13 @@ type Probes struct {
// half-working this field exists to end. nil is "never asked / couldn't
// tell" and publishes as null, same as every other probe here.
ConfigPathRunning *bool
+ // RestartPathRunning is whether this box's restart trigger unit
+ // (carlos-restart.path) is ACTIVE — ConfigPathRunning's issue #113 sibling,
+ // same failure mode, sharper consequence: with it unarmed, a member who
+ // runs `carlos restart` against a wedged instance is told "requested" and
+ // waits out the hourly pass for a cycle they were led to expect in
+ // seconds. nil is "never asked / couldn't tell" and publishes as null.
+ RestartPathRunning *bool
// ExpiredCanaries is how many of this box's own canary/* routes have
// passed release.CanaryTTL (pipelines spec §5 "Canaries expire"),
// pre-computed by the caller against the release store — CollectBox
@@ -171,6 +178,14 @@ type Box struct {
// issue #44 carry no such key at all and decode to null, which is the
// honest reading of a box that never reported it.
ConfigPathRunning *bool `json:"config_path_running"`
+ // RestartPathRunning is whether a member's `carlos restart` reaches this
+ // box's units in seconds (carlos-restart.path armed) or only on the hourly
+ // floor: true = armed, false = installed but not watching, null = unknown.
+ // Same content and same privacy argument as ConfigPathRunning above — the
+ // liveness of one of the platform's own units on an opaquely-labelled box.
+ // Snapshots written before issue #113 carry no such key and decode to null,
+ // which is the honest reading of a box that never reported it.
+ RestartPathRunning *bool `json:"restart_path_running"`
// ExpiredCanaries publishes Probes.ExpiredCanaries straight through — a
// count, never hostnames (pipelines spec §5): which canary route expired
// is exactly the kind of identifying detail this package's whole design
@@ -353,16 +368,17 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {
}
box := Box{
- GeneratedAt: now,
- Label: Label(p.MachineID),
- Region: p.Region,
- Routes: counts,
- Load: bandWord(p.MemFrac, loadWords),
- Capacity: bandWord(worst, capacityWords),
- Replication: Replication{Healthy: healthy, LagSecs: p.LagSecs},
- AdoptionRunning: p.AdoptRunning,
- ConfigPathRunning: p.ConfigPathRunning,
- ExpiredCanaries: p.ExpiredCanaries,
+ GeneratedAt: now,
+ Label: Label(p.MachineID),
+ Region: p.Region,
+ Routes: counts,
+ Load: bandWord(p.MemFrac, loadWords),
+ Capacity: bandWord(worst, capacityWords),
+ Replication: Replication{Healthy: healthy, LagSecs: p.LagSecs},
+ AdoptionRunning: p.AdoptRunning,
+ ConfigPathRunning: p.ConfigPathRunning,
+ RestartPathRunning: p.RestartPathRunning,
+ ExpiredCanaries: p.ExpiredCanaries,
}
// Metrics attaches only when something is known — an all-unknown Probes
diff --git a/internal/status/status_test.go b/internal/status/status_test.go
index 928b9d609..4e83e905b 100644
--- a/internal/status/status_test.go
+++ b/internal/status/status_test.go
@@ -758,6 +758,53 @@ func TestConfigPathRunningIsWireAdditive(t *testing.T) {
}
}
+// TestRestartPathRunningTravelsPerBox is TestConfigPathRunningTravelsPerBox's
+// issue #113 twin, and it exists for a sharper reason than symmetry: with
+// carlos-restart.path unarmed, a member runs `carlos restart` at a wedged
+// instance, is told "requested", and waits for a cycle that will not come until
+// the hourly pass. Nothing inside the agent can see that — it writes its marker
+// either way — so this reading is the only thing that can. Unknown stays null.
+func TestRestartPathRunningTravelsPerBox(t *testing.T) {
+ if b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()); b.RestartPathRunning != nil {
+ t.Errorf("restart_path_running = %v, want null when the probe never answered", *b.RestartPathRunning)
+ }
+ for _, want := range []bool{true, false} {
+ got := status.CollectBox(nil, status.Probes{MachineID: "m", RestartPathRunning: &want}, time.Now()).RestartPathRunning
+ if got == nil || *got != want {
+ t.Fatalf("RestartPathRunning=%v: box field = %v, want %v", want, got, want)
+ }
+ }
+
+ s := teststore(t)
+ now := time.Now()
+ no := false
+ b := status.CollectBox(nil, status.Probes{MachineID: "m", RestartPathRunning: &no}, now)
+ if err := status.WriteBox(context.Background(), s, b); err != nil {
+ t.Fatal(err)
+ }
+ f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(f.Boxes) != 1 {
+ t.Fatalf("fleet boxes = %d, want 1", len(f.Boxes))
+ }
+ if got := f.Boxes[0].RestartPathRunning; got == nil || *got {
+ t.Fatalf("fleet box restart_path_running = %v, want false — knowing WHICH box cannot cycle a wedged instance is the whole value", got)
+ }
+
+ // Wire-additive, like every field before it: a snapshot written before
+ // issue #113 has no such key and must read as unknown, not a healthy false.
+ var old status.Box
+ raw := `{"generated_at":"2026-08-04T00:00:00Z","label":"box-old","routes":{"static":0,"resident":0,"hibernating":0,"awake":null,"asleep":null},"load":null,"capacity":null,"replication":{"healthy":null,"lag_seconds":null},"adoption_running":true,"config_path_running":true,"expired_canaries":null}`
+ if err := json.Unmarshal([]byte(raw), &old); err != nil {
+ t.Fatalf("a pre-#113 snapshot must still parse: %v", err)
+ }
+ if old.RestartPathRunning != nil {
+ t.Fatalf("restart_path_running = %v, want null for a snapshot written before the field existed", *old.RestartPathRunning)
+ }
+}
+
// TestFleetAdoptionWorstOf covers the aggregate: a box that is KNOWN not to be
// adopting is a defect the fleet number must never hide, so one false outranks
// any number of trues; an unknown box can't be summed into a confident true,
diff --git a/main.go b/main.go
index 18735f150..58b38f113 100644
--- a/main.go
+++ b/main.go
@@ -59,6 +59,8 @@ func main() {
runPromote(args)
case "deploy":
runDeploy(args)
+ case "restart":
+ runRestart(args)
case "channels":
runChannels(args)
case "releases":
@@ -119,6 +121,7 @@ Your apps — ship and run software on a CARLOS deployment
promote move a version up the ladder: canary → edge → beta → stable
(reaching stable cuts the release's semver tag)
deploy ship + promote + wait until the URL serves the new version
+ restart cycle an app's instances — no new version, no config change
rollback point a channel back at an earlier version
channels show what each of an app's channels is serving
releases every version shipped, newest first, with labels and tags
diff --git a/restart.go b/restart.go
new file mode 100644
index 000000000..25c9e5a31
--- /dev/null
+++ b/restart.go
@@ -0,0 +1,86 @@
+// restart.go is `carlos restart`: cycle an app's processes with no version
+// and no config change (issue #113). It is the member-facing answer to a
+// wedged instance, which until now meant `systemctl` on a box — and every
+// carlos command is designed for someone with no box access at all.
+//
+// Pull-based, like everything else here: the console touches one small object,
+// every box serving the app notices within seconds and cycles its own routes.
+// Nothing is pushed at a box, nothing wakes a sleeping instance, and no
+// release ladder or bake window is involved — there is no new version for one
+// to gate.
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "log"
+ "strings"
+)
+
+// runRestart is the verb. API mode ONLY, like deploy: the request is a console
+// write, and the object it touches means nothing without a fleet reading it.
+func runRestart(args []string) {
+ fs := flag.NewFlagSet("restart", flag.ExitOnError)
+ app := fs.String("app", "", "app whose instances to cycle")
+ acctRaw, acctIDRaw, _ := accountFlags(fs)
+ consoleFlag := fs.String("console", "", "console origin to restart through (API mode only; default: the only logged-in one)")
+ fs.Parse(args)
+ name := strings.TrimSpace(*app)
+ if name == "" {
+ log.Fatal("usage: carlos restart -app <a> [-account <sqid-or-name>] [-account-id <sqid>] [-console <origin>]")
+ }
+ // Client-side gate before the server's, the same one `carlos apps create`
+ // applies: a name the console is certain to refuse should cost a typo
+ // nothing, and the shared write budget is spent per attempt.
+ if err := validAppName(name); err != nil {
+ log.Fatalf("restart: %v", err)
+ }
+
+ sess := requireRestartAPISession(*consoleFlag)
+ acct := apiAccount(acctRaw, acctIDRaw)
+ printUsing(stdout, sess)
+ res, err := sess.restart(context.Background(), acct, name)
+ if err != nil {
+ log.Fatalf("restart: %v", err)
+ }
+ // One line, and it says REQUESTED rather than restarted, because that is
+ // what actually happened: the boxes converge on their own within seconds,
+ // and this terminal is not watching them do it. Claiming a restart here
+ // would be the same lie issue #110 was made of, told by the CLI instead of
+ // a header.
+ //
+ // It also does not promise that everything is back up in seconds, because
+ // that is not how the box behaves: long-running tenants are restarted
+ // within seconds, while exec-backed instances are STOPPED within seconds
+ // and respawn on their next request or scheduled run — which for an idle
+ // app may be a while, and is the correct behaviour, not a delay to
+ // apologise for.
+ fmt.Fprintf(stdout, "%s for %s/%s — running instances stop within seconds; "+
+ "long-running ones restart, on-demand ones respawn on their next request "+
+ "(sleeping instances stay asleep and start fresh when next woken)\n",
+ colorFor(stdout).ok("restart requested"), res.Account, res.App)
+}
+
+// requireRestartAPISession resolves signed-in mode for restart, refusing
+// direct-bucket mode with restart's OWN message — the pattern
+// requireDeployAPISession sets, and for its reason: a refusal that names
+// another verb's advice sends people somewhere that cannot help them.
+//
+// There is deliberately no headless equivalent to fall back to. A bucket-mode
+// operator asking for a restart already has the box (that is what bucket mode
+// means), so the honest answer is the box's own systemctl — not a bucket write
+// this CLI could make and no console would announce.
+func requireRestartAPISession(consoleFlag string) *apiSession {
+ sess, err := resolveMode(consoleFlag)
+ if err != nil {
+ log.Fatal(err)
+ }
+ if sess == nil {
+ log.Fatal("carlos restart works through a console, and this shell is in direct bucket mode\n" +
+ "(CARLOS_DEPLOYMENT_BUCKET/CARLOS_DEPLOYMENT_DIR is set). restart is a member verb with no\n" +
+ "direct-bucket equivalent — the request is a console write the fleet listens for. Unset the\n" +
+ "store env and run: carlos auth login -console <url>.")
+ }
+ return sess
+}
diff --git a/restart_test.go b/restart_test.go
new file mode 100644
index 000000000..8232da794
--- /dev/null
+++ b/restart_test.go
@@ -0,0 +1,153 @@
+package main
+
+// The CLI half of `carlos restart` (issue #113): the wire shape it sends, the
+// answers it can get, and the one refusal it has to explain rather than
+// repeat.
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os/exec"
+ "strings"
+ "testing"
+)
+
+// TestAPISessionRestartWireShape pins what handleAPIRestart requires: a bearer
+// POST to the app's /restart path, with the account segment carried verbatim
+// (the console resolves names against the caller's own memberships — this
+// client must never resolve one locally).
+func TestAPISessionRestartWireShape(t *testing.T) {
+ var gotPath, gotMethod, gotAuth string
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotPath, gotMethod, gotAuth = r.URL.RequestURI(), r.Method, r.Header.Get("Authorization")
+ json.NewEncoder(w).Encode(map[string]any{"requested": true, "account": "bab", "app": "keymail"})
+ }))
+ defer ts.Close()
+
+ res, err := newTestSession(ts.URL).restart(context.Background(), apiAcct{Seg: "keymail-team"}, "keymail")
+ if err != nil {
+ t.Fatalf("restart: %v", err)
+ }
+ if gotMethod != http.MethodPost {
+ t.Fatalf("method = %s, want POST", gotMethod)
+ }
+ if gotPath != "/api/cli/apps/keymail-team/keymail/restart" {
+ t.Fatalf("path = %s", gotPath)
+ }
+ if gotAuth != "Bearer cli-token" {
+ t.Fatalf("auth = %q", gotAuth)
+ }
+ // The CANONICAL names come back from the console, not from what was typed.
+ if res.Account != "bab" || res.App != "keymail" {
+ t.Fatalf("result = %+v, want the console's canonical bab/keymail", res)
+ }
+}
+
+// TestAPISessionRestartAccountIDIsExact: -account-id travels as ?exact=sqid,
+// the console-side switch that matches the segment as a sqid ONLY — the
+// reference an ambiguity error tells you to reach for must not itself be
+// ambiguous.
+func TestAPISessionRestartAccountIDIsExact(t *testing.T) {
+ var gotPath string
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotPath = r.URL.RequestURI()
+ json.NewEncoder(w).Encode(map[string]any{"requested": true, "account": "bab", "app": "keymail"})
+ }))
+ defer ts.Close()
+
+ if _, err := newTestSession(ts.URL).restart(context.Background(), apiAcct{Seg: "bab", Exact: true}, "keymail"); err != nil {
+ t.Fatalf("restart: %v", err)
+ }
+ if !strings.HasSuffix(gotPath, "?exact=sqid") {
+ t.Fatalf("path = %s, want the exact=sqid switch", gotPath)
+ }
+}
+
+// TestAPISessionRestart404NamesBothCauses is the old-console case. net/http's
+// mux answers an unregistered route with the SAME byte-identical "404 page not
+// found" apiAccountApp gives a non-member, so there is nothing in the response
+// to tell them apart — and a message that claimed either one alone would send
+// somebody to fix the wrong thing.
+func TestAPISessionRestart404NamesBothCauses(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.NotFound(w, r) // a console that predates this door, or a stranger's probe
+ }))
+ defer ts.Close()
+
+ _, err := newTestSession(ts.URL).restart(context.Background(), apiAcct{Seg: "bab"}, "keymail")
+ if err == nil {
+ t.Fatal("a 404 must error")
+ }
+ if !strings.Contains(err.Error(), "-account") {
+ t.Fatalf("error %q drops the membership explanation", err)
+ }
+ if !strings.Contains(err.Error(), "predates") || !strings.Contains(err.Error(), "redeploy the console") {
+ t.Fatalf("error %q does not name the old-console cause and its fix", err)
+ }
+}
+
+// TestAPISessionRestartServerErrorSurfacesVerbatim: a door that answered with
+// a real diagnosis (a 429 budget, a 503 with no bucket, a 502 that wrote
+// nothing) must reach the terminal in the console's own words — that text
+// already says whether a retry is safe.
+func TestAPISessionRestartServerErrorSurfacesVerbatim(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusTooManyRequests)
+ json.NewEncoder(w).Encode(map[string]string{"error": "too many requests — wait a minute and try again"})
+ }))
+ defer ts.Close()
+
+ _, err := newTestSession(ts.URL).restart(context.Background(), apiAcct{Seg: "bab"}, "keymail")
+ if err == nil || !strings.Contains(err.Error(), "wait a minute") {
+ t.Fatalf("restart 429 = %v, want the console's own text", err)
+ }
+}
+
+// TestRestartVerbRefusals drives the real binary for the two ways the verb
+// stops before it ever dials anything: no -app, and direct bucket mode. Both
+// run through carlosTestEnv with a scratch cmd.Dir — the repo rule — so a
+// logged-in laptop's real credential and console can never be what this test
+// exercises.
+func TestRestartVerbRefusals(t *testing.T) {
+ bin := buildCarlos(t)
+ scratch := t.TempDir()
+ run := func(env []string, args ...string) (string, error) {
+ t.Helper()
+ cmd := exec.Command(bin, args...)
+ cmd.Env = carlosTestEnv(t, env...)
+ cmd.Dir = scratch
+ out, err := cmd.CombinedOutput()
+ return string(out), err
+ }
+
+ out, err := run(nil, "restart")
+ if err == nil {
+ t.Fatalf("restart with no -app exited 0:\n%s", out)
+ }
+ if !strings.Contains(out, "usage: carlos restart -app") {
+ t.Fatalf("restart with no -app = %q, want the usage line", out)
+ }
+
+ // Direct bucket mode: refused in restart's OWN words, naming the login
+ // that fixes it — never another verb's advice.
+ out, err = run([]string{"CARLOS_DEPLOYMENT_DIR=" + t.TempDir()}, "restart", "-app", "keymail")
+ if err == nil {
+ t.Fatalf("restart in direct bucket mode exited 0:\n%s", out)
+ }
+ if !strings.Contains(out, "carlos restart works through a console") ||
+ !strings.Contains(out, "carlos auth login") {
+ t.Fatalf("direct-mode refusal = %q, want restart's own message and the login fix", out)
+ }
+
+ // A malshaped app name is refused at the terminal, before the console's
+ // shared write budget is spent on a request it is certain to reject.
+ out, err = run(nil, "restart", "-app", "Not An App")
+ if err == nil {
+ t.Fatalf("restart with a bad app name exited 0:\n%s", out)
+ }
+ if !strings.Contains(out, "not a usable app name") {
+ t.Fatalf("bad app name = %q, want the name gate", out)
+ }
+}