amadan

carlos / platform Public

Merge branch 'restart-selectors' into main

claude pushed by paul@keymail.dev 3efad49dd3408e4338ab49232a7d264dae0374fa parent 5e6f75eparent 38b1a83
17 files changed, +3018 −56
  • AGENTS.md +17 −0
  • apimode.go +62 −0
  • docs/site/cli.md +50 −1
  • internal/console/apicli.go +9 −6
  • internal/console/apirestartscope.go +390 −0
  • internal/console/apirestartscope_test.go +422 −0
  • internal/console/authcli.go +41 −1
  • internal/host/agent.go +9 −7
  • internal/host/restart.go +167 −21
  • internal/host/restart_test.go +34 −13
  • internal/host/restartscope.go +289 −0
  • internal/host/restartscope_test.go +577 −0
  • internal/release/restartscope.go +314 −0
  • internal/release/restartscope_test.go +301 −0
  • main.go +2 −1
  • restart.go +152 −6
  • restart_test.go +182 −0

CI passed — run details

diff --git a/AGENTS.md b/AGENTS.md
index 20a55f2b3..f50e5801a 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -278,3 +278,20 @@ anything.
PR #10). Hibernating instances are never woken. Static routes carry
`X-Carlos-Version` from the same roll (#112), so `carlos deploy -kind
static -host <site>` verifies static sites too.
+- **`carlos restart --host` / `--environment` / `--dry-run`** (discussion
+ #16): the same verb narrowed to one route, or to everything reading one
+ config bundle — the shape that follows `carlos secrets set --environment
+ relay`, since a bundle is delivered to an ENVIRONMENT. `--host` wins over
+ `--environment`, and the console refuses the pair when that host does not
+ read that environment; a selector matching nothing is refused with what
+ the app actually has, never widened. The narrowed request is a SECOND
+ bucket object (`apps/<acct>/<app>/restart-scope.json`, an append log of
+ `{id, requested_at, scope}`) reached through a SECOND console path
+ (`POST .../restart/scope`), because both old readers act on the app-wide
+ object alone: an old box never reads the log and an old console answers
+ 404, so neither can turn a narrowed request into a whole-app restart. A
+ box steps OVER a selector `kind` it does not know rather than falling
+ back to the app — never widen on ignorance. Boxes remember the last
+ request id they acted on (`scopes` in `restart-marks.json`, additive in
+ both directions); a mark that has rolled off the log re-baselines and
+ cycles nothing.
diff --git a/apimode.go b/apimode.go
index 1c7aebe66..7cbb84f8c 100644
--- a/apimode.go
+++ b/apimode.go
@@ -1044,6 +1044,68 @@ func (s *apiSession) restart(ctx context.Context, a apiAcct, app string) (apiRes
return res, nil
}
+// apiRestartScopeRoute is one route a narrowed restart addressed — the
+// console's answer to "what exactly is about to stop". Hibernate is not
+// decoration: a hibernating route is named AND left asleep, and an operator
+// who sees it listed with no explanation would read that as a failure.
+type apiRestartScopeRoute struct {
+ Host string `json:"host"`
+ Environment string `json:"environment"`
+ Unit string `json:"unit"`
+ Hibernate bool `json:"hibernate"`
+}
+
+// apiRestartScopeResult is what the narrowed door answers with. Requested is
+// FALSE for a dry run and true otherwise, so the CLI never has to infer from
+// its own flags what the console did — the console says.
+type apiRestartScopeResult struct {
+ Account string `json:"account"`
+ App string `json:"app"`
+ Scope struct {
+ Kind string `json:"kind"`
+ Value string `json:"value"`
+ } `json:"scope"`
+ DryRun bool `json:"dry_run"`
+ Requested bool `json:"requested"`
+ Routes []apiRestartScopeRoute `json:"routes"`
+}
+
+// restartScoped asks the console to cycle ONE route, or everything reading one
+// config environment — or, with dryRun, to say what that would be and touch
+// nothing (discussion #16).
+//
+// A separate path from restart() above, not a body on it, and the 404 message
+// is where that shows. The old door reads no body at all: a narrowed request
+// sent there would be answered by restarting every instance the app owns,
+// which is the accident this exists to prevent. So the CLI asks a path an old
+// console does not serve, gets net/http's byte-identical "404 page not found",
+// and says all three things it could mean — shortest fix first — the way
+// restart() widens its own.
+func (s *apiSession) restartScoped(ctx context.Context, a apiAcct, app, host, environment string, dryRun bool) (apiRestartScopeResult, error) {
+ in := struct {
+ Host string `json:"host"`
+ Environment string `json:"environment"`
+ DryRun bool `json:"dry_run"`
+ }{host, environment, dryRun}
+ status, body, err := s.postJSON(ctx, apiPath(a, app, "/restart/scope"), in)
+ if err != nil {
+ return apiRestartScopeResult{}, err
+ }
+ if status != http.StatusOK {
+ if status == http.StatusNotFound {
+ return apiRestartScopeResult{}, fmt.Errorf("%w — or this console predates narrowed restarts "+
+ "(--host, --environment, --dry-run), in which case redeploy the console first; "+
+ "`carlos restart --app` still works on it, but cycles every instance the app has", notFoundErr())
+ }
+ return apiRestartScopeResult{}, apiErrorFromBody(status, body)
+ }
+ var res apiRestartScopeResult
+ if err := json.Unmarshal(body, &res); err != nil {
+ return apiRestartScopeResult{}, fmt.Errorf("restart: unreadable response: %w", err)
+ }
+ return res, nil
+}
+
// apiScheduleList mirrors handleAPIScheduleList's wire shape
// (internal/console/apischedule.go's scheduleAnswer) byte for byte — this
// binary never imports internal/console, so the shape is redeclared here
diff --git a/docs/site/cli.md b/docs/site/cli.md
index 2fcf1f22e..3b017d0d6 100644
--- a/docs/site/cli.md
+++ b/docs/site/cli.md
@@ -234,7 +234,7 @@ percentage of production traffic on the app's own hostname.
### carlos restart
-Cycles an app's processes with no new version and no config change. This is
+Cycles running processes with no new version and no config change. This is
the answer to a wedged instance.
```sh
@@ -248,6 +248,55 @@ stop within seconds and respawn on their next request, which for an idle app
may be a while. Nothing is stuck: the instance comes back with the next
request that needs it. Hibernating instances are left asleep.
+#### Restarting less than the whole app
+
+An app is not always a small thing. One app can own a relay, a router, a
+tunnel and every customer instance — so "restart the app" can mean cycling all
+of production to pick up one credential that reached one route. Two flags
+narrow it:
+
+```sh
+carlos restart --app titogo --host bill.example.com
+carlos restart --app titogo --environment relay
+```
+
+`--host` cycles exactly that route. `--environment` cycles every route reading
+that named config bundle, which is the one to reach for after a
+[carlos secrets set](/docs/cli#carlos-secrets) or
+[carlos env set](/docs/cli#carlos-env) on the same `--environment`: a bundle is
+delivered to an environment, so "restart what reads this" is the request that
+follows it.
+
+Both narrowed forms print the routes they addressed — host, environment, the
+unit that owns the process where there is one, and which of them were asleep
+and so left alone. `--app` on its own still prints the single line it always
+has; `--dry-run` is how you see its set.
+
+```sh
+carlos restart --app titogo --environment relay --dry-run
+```
+
+A dry run resolves the selector, names every route it would cycle, and stops
+there. Nothing is written and nothing restarts.
+
+Precedence, when you give both: `--host` wins, but only after the console
+checks that the host really does read that environment. If it does not, the
+command refuses rather than picking one — one of the two flags is a typo, and
+this is not a verb to guess on. A selector that matches nothing is refused the
+same way, and the refusal lists the hosts or environments the app actually
+has. Nothing is ever widened back to the whole app.
+
+Two limits worth knowing. The console resolves the selector against its own
+box's registry, and it refuses a selector nothing there matches — so on a
+multi-box fleet you can only ask for a `--host` or an `--environment` that at
+least one route on the console's own box reads, and the printed set is what
+that box can see. A request it does accept is a bucket object every box
+converges on, and each box applies the same selector to its own routes; those
+other routes are cycled without appearing in the list. And a console or a box
+that predates these flags does not understand a narrowed request: the console
+answers that it has no such door, and a box that has not been rolled yet simply
+does nothing. Neither one turns it into a restart of everything.
+
### carlos schedule
Gives an app a timetable. A schedule is a time and a path: at each fire the
diff --git a/internal/console/apicli.go b/internal/console/apicli.go
index 3fad61330..e6e2613b5 100644
--- a/internal/console/apicli.go
+++ b/internal/console/apicli.go
@@ -1071,12 +1071,15 @@ func (s *Server) handleAPIRollback(w http.ResponseWriter, r *http.Request) {
// path — there is no restart "options" — and a bearer request carries no
// cookie for a cross-site form to forge.
//
-// No per-host scoping. The generation is per APP, so every box serving it
-// cycles its own routes for that app; a -host flag would need a per-host
-// object and a way to say what happens to the hosts it doesn't name, and
-// "restart my app" is the request people actually have. Boxes skip their
-// hibernating routes on their own (internal/host/restart.go) — nothing here
-// wakes anything.
+// No per-host scoping HERE, and that is now a division of labour rather than
+// a limit. This door's generation is per APP: every box serving it cycles its
+// own routes for that app, which is what makes it the one form every box in
+// the fleet understands, including the ones that predate narrowing. Narrowed
+// requests (`--host`, `--environment`) live on their own path and their own
+// object — apirestartscope.go, discussion #16 — precisely so that a box or a
+// console that cannot understand a narrowing does NOTHING rather than falling
+// back to this. Boxes skip their hibernating routes on their own
+// (internal/host/restart.go) — nothing here wakes anything.
func (s *Server) handleAPIRestart(w http.ResponseWriter, r *http.Request) {
addr, acct, app, ok := s.apiAccountApp(w, r)
if !ok {
diff --git a/internal/console/apirestartscope.go b/internal/console/apirestartscope.go
new file mode 100644
index 000000000..6ce6f3fb2
--- /dev/null
+++ b/internal/console/apirestartscope.go
@@ -0,0 +1,390 @@
+// The NARROWED restart door (discussion #16): `carlos restart --host` and
+// `carlos restart --environment`, plus the dry run that shows an operator the
+// blast radius before anything stops.
+//
+// It is a SEPARATE PATH from handleAPIRestart, and that is a safety property
+// rather than tidiness. The old door reads no body at all, so a narrowed
+// request sent to an old console would be answered by cycling every instance
+// the app owns — the exact accident this feature exists to prevent, committed
+// by the server instead of the box. A distinct path cannot be misread: a
+// console that predates this answers 404, the CLI says so and nothing is
+// restarted. The rule is the same one internal/release/restartscope.go states
+// for boxes — a reader that cannot understand a narrowing must do NOTHING,
+// never something wider.
+//
+// The doors, in order, are handleAPIRestart's with two additions:
+//
+// token — cliUser's one 401, before any name is resolved.
+// account+app 404 — apiAccountApp, so probing teaches a stranger nothing.
+// body shape — 400 for a field this door does not declare. A renamed
+// or misspelled key that was merely IGNORED would leave a
+// body with no selector in it, and no selector means the
+// whole app.
+// selector shape — 400 for a malformed host or environment, before the
+// registry is opened.
+// registry — 503 when this console serves none. A narrowed request
+// cannot be resolved OR reported without one, and the one
+// thing this door must never do is answer "requested" for
+// a selector it did not check.
+// resolution — 400 when the selector names nothing, LISTING what the
+// app actually has. A typo in --environment that fell
+// through to an app-wide restart would be a worse version
+// of the bug being fixed.
+// narrowing — 400 for a WRITE with no selector at all. The app-wide
+// case is answered here only as a --dry-run, which is how
+// somebody sees the blast radius before typing the real
+// verb; performing it belongs to the app-wide door, whose
+// object every box in the fleet already reads.
+// rate limit — the shared `write` budget, keyed on the address, and
+// only for a request that WRITES. A dry run is a read of
+// this box's own registry, the same read `carlos routes`
+// makes through a door with no budget at all.
+// write + bump — the request object, then the announcement, in that
+// order.
+//
+// SCOPE: like handleAPIRoutes and the config door's restart notice, this
+// console reads its OWN box's registry, so in a multi-box fleet the other
+// boxes' routes are not represented in what it resolves or reports. Once a
+// request is accepted it does reach them — it is a bucket object every box
+// converges on — and each box matches it against its own routes, which is what
+// makes `--environment relay` mean "everything reading that bundle"
+// fleet-wide.
+//
+// The limit that cuts the other way, and is worth stating plainly because the
+// resolution gate above is what causes it: a selector NO route on this box
+// matches is refused, so a --host or an --environment that only another box in
+// the fleet knows about cannot be requested here at all. That direction is
+// safe — a refusal, never a wider restart — but it is a real limit, not just a
+// reporting one, and docs/site/cli.md says so in the same words.
+package console
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "net/http"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// restartScopeMaxBytes bounds the request body. Two short strings and a bool;
+// anything larger is a client that has misunderstood the door.
+const restartScopeMaxBytes = 4 << 10
+
+// restartScopeNamesShown caps how many hosts or environments a refusal lists.
+// A refusal exists to let someone spot their typo, and nineteen names do that
+// where two hundred would not — the count that follows carries the rest.
+const restartScopeNamesShown = 20
+
+// apiRestartRoute is one addressed route in the answer. Environment and Unit
+// are the two columns an operator reads to check the set is the one they
+// meant; Hibernate is the one that says a named route will NOT cycle now, and
+// why that is correct rather than a failure.
+type apiRestartRoute struct {
+ Host string `json:"host"`
+ Environment string `json:"environment,omitempty"`
+ Unit string `json:"unit,omitempty"`
+ Hibernate bool `json:"hibernate"`
+}
+
+// handleAPIRestartScope answers `carlos restart --host/--environment` and any
+// `--dry-run`.
+func (s *Server) handleAPIRestartScope(w http.ResponseWriter, r *http.Request) {
+ addr, acct, app, ok := s.apiAccountApp(w, r)
+ if !ok {
+ return
+ }
+ var req struct {
+ Host string `json:"host"`
+ Environment string `json:"environment"`
+ DryRun bool `json:"dry_run"`
+ }
+ // STRICT, and that is a safety property rather than pedantry: a key this
+ // door does not declare — `"hosts"` for `"host"`, a field a future client
+ // renamed — would otherwise be dropped, leaving a body with no selector in
+ // it, which resolves to the whole app. A door that exists because the other
+ // door widens must not widen on a typo of its own.
+ if !readJSONStrictLimit(w, r, &req, restartScopeMaxBytes) {
+ return
+ }
+ host := strings.TrimSpace(req.Host)
+ envName := strings.TrimSpace(req.Environment)
+ if host != "" && !provision.ValidHostShape(host) {
+ jsonError(w, http.StatusBadRequest, fmt.Sprintf("%q is not a usable hostname", host))
+ return
+ }
+ // The config doors' own gate, reused rather than restated: one shape rule
+ // for one flag, including its refusal of the literal "default" (which
+ // names the base bundle no route is ever BOUND to — a restart of what
+ // reads that is a restart of the app).
+ if !validAPIEnvironment(w, envName) {
+ return
+ }
+ if s.cfg.Registry == "" {
+ // The app-wide door still works on a console like this, because it
+ // resolves nothing. This one cannot: it would have to answer
+ // "requested" for a selector it never checked and a set it cannot
+ // name, which is the whole thing the verb was narrowed to avoid.
+ // (Opening "" would also hand sqlite a throwaway temp DB and answer
+ // "no routes" having created the file it said that from —
+ // handleAPIRoutes' twin guard.)
+ jsonError(w, http.StatusServiceUnavailable,
+ "this console serves no route registry, so it cannot tell which routes a narrowed restart would address — "+
+ "use carlos restart --app to cycle the whole app, or run this against the console on the box")
+ return
+ }
+ reg, err := openRegistry(s.cfg.Registry)
+ if err != nil {
+ log.Printf("console: api restart scope: registry %s: %v", s.cfg.Registry, err)
+ jsonError(w, http.StatusBadGateway, "could not read the route registry — nothing was cycled; try again")
+ return
+ }
+ all, err := reg.List()
+ if err != nil {
+ log.Printf("console: api restart scope: registry list: %v", err)
+ jsonError(w, http.StatusBadGateway, "could not read the route registry — nothing was cycled; try again")
+ return
+ }
+ // The tenant boundary, and the only one: stamped to THIS app and resolving
+ // to THIS account, through release.RouteAccount rather than a bare column
+ // compare — a second private notion of "whose route is this" is how a
+ // listing ends up disagreeing with the reconciler about ownership.
+ // release.Adoptable on top, because that is the exact predicate the box's
+ // restart passes use: a route no pass would adopt is not one whose process
+ // this verb has any business claiming to cycle.
+ var mine []registry.Route
+ for _, rt := range all {
+ if rt.App != app || release.RouteAccount(rt) != acct.Sqid || !release.Adoptable(rt) {
+ continue
+ }
+ mine = append(mine, rt)
+ }
+ scope, addressed, refusal := resolveRestartScope(mine, host, envName)
+ if refusal != "" {
+ jsonError(w, http.StatusBadRequest, refusal)
+ return
+ }
+ // An app-wide request may be DESCRIBED here and never PERFORMED here. The
+ // dry run is the whole reason the app-wide case reaches this door at all —
+ // it is how somebody sees the blast radius before typing the real thing —
+ // but a write with no selector in it would cycle every route the app owns
+ // through the very door that exists because the other one widens. The
+ // app-wide restart has its own door, which every box in the fleet
+ // understands; this one only ever narrows.
+ if scope.Kind == release.RestartScopeApp && !req.DryRun {
+ jsonError(w, http.StatusBadRequest,
+ "this door only narrows a restart: name --host or --environment, or add --dry-run to see what a whole-app "+
+ "restart would cycle. To actually cycle the whole app, run: carlos restart --app "+app)
+ return
+ }
+ answer := map[string]any{
+ "account": acct.Sqid, "app": app,
+ "scope": map[string]string{"kind": scope.Kind, "value": scope.Value},
+ "dry_run": req.DryRun,
+ "requested": false,
+ "routes": restartRouteRows(addressed),
+ }
+ if req.DryRun {
+ // Nothing written, nothing bumped, nothing audited: this is a read of
+ // the registry with a selector applied, and the honest answer to
+ // "what would this cycle" must not itself cycle anything.
+ writeJSON(w, http.StatusOK, answer)
+ return
+ }
+ p, _ := principalFrom(r.Context())
+ if !s.allowWrite(p, time.Now()) {
+ jsonError(w, http.StatusTooManyRequests, "too many requests — wait a minute and try again")
+ return
+ }
+ if s.cfg.Releases == nil {
+ jsonError(w, http.StatusServiceUnavailable,
+ "no deployment bucket on this host — this console cannot request a restart")
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
+ defer cancel()
+ if err := s.writeRestartScope(ctx, acct.Sqid, app, scope); err != nil {
+ // Logged, not returned: a store error carries bucket names, endpoints
+ // and request ids. The caller gets the fixed sentence every other
+ // bucket failure here gets, and a retry is safe — one more request is
+ // one more cycle of the same routes.
+ log.Printf("console: api restart scope %s/%s (%s %s): %v", acct.Sqid, app, scope.Kind, scope.Value, err)
+ jsonError(w, http.StatusBadGateway, "could not request the restart — nothing was cycled; try again")
+ return
+ }
+ s.audit(acct, app, addr, "restart", restartAuditWords(scope))
+ // The request is in the bucket; this is what makes boxes LOOK at it — they
+ // read restart requests only when the serial moves.
+ s.bumpSerial(ctx)
+ answer["requested"] = true
+ writeJSON(w, http.StatusOK, answer)
+}
+
+// restartAuditWords is the audit row's sentence. It names the SELECTOR, not
+// the resolved hosts: the row is a record of what somebody asked for, and the
+// set it resolved to is a property of one box's registry at one moment, which
+// would read as a claim about the fleet the moment a route moved.
+//
+// The app-wide branch is kept although the handler no longer lets an app-wide
+// WRITE reach the audit at all: if a later edit ever restores one, the row it
+// writes must read as the app-wide door's own row rather than as
+// "requested a restart of app (cli)".
+func restartAuditWords(scope release.RestartScope) string {
+ if scope.Kind == release.RestartScopeApp {
+ return "requested a restart (cli)" // the app-wide door's own wording
+ }
+ return fmt.Sprintf("requested a restart of %s %s (cli)", scope.Kind, scope.Value)
+}
+
+// writeRestartScope puts the request where the boxes will find it: the log
+// only new boxes read, which is what keeps an old box from widening a request
+// it cannot understand into a whole-app restart.
+//
+// An app-wide scope is REFUSED here rather than forwarded to the old object,
+// which is belt to the handler's braces. The handler has already turned that
+// body away, and this is the check that keeps the refusal true if a later edit
+// moves or loses it: an app-wide restart is the other door's write, and a
+// second way to reach it through the narrowing door is exactly the accident
+// this file was built to prevent. The caller answers a store error with "could
+// not request the restart — nothing was cycled", which is the honest outcome.
+func (s *Server) writeRestartScope(ctx context.Context, acct, app string, scope release.RestartScope) error {
+ if scope.Kind != release.RestartScopeHost && scope.Kind != release.RestartScopeEnvironment {
+ return fmt.Errorf("restart scope: %q is not a narrowing — an app-wide restart belongs to the app-wide door", scope.Kind)
+ }
+ _, err := release.AppendRestartScope(ctx, s.cfg.Releases, acct, app, scope)
+ return err
+}
+
+// resolveRestartScope turns the two flags into ONE selector and the routes it
+// addresses, or into the sentence that refuses them.
+//
+// PRECEDENCE, and the reason it is this and not "narrowest silently wins":
+//
+// - --host alone selects that one route.
+// - --environment alone selects every route bound to that bundle — what
+// "restart what reads this" means, and the shape that pairs with
+// `carlos secrets set --environment`.
+// - both: --host wins, but only after the named host is CHECKED against the
+// named environment. Quietly ignoring the wider flag would hide the case
+// where one of the two is a typo, and this verb's whole job is telling an
+// operator what they are about to cycle.
+// - neither: the app, exactly as before.
+//
+// A selector that names nothing is refused with what the app actually has —
+// never widened. An --environment typo that fell through to an app-wide
+// restart is the nineteen-route accident this door was built to prevent.
+func resolveRestartScope(routes []registry.Route, host, envName string) (release.RestartScope, []registry.Route, string) {
+ switch {
+ case host != "":
+ rt, ok := routeWithHost(routes, host)
+ if !ok {
+ return release.RestartScope{}, nil, fmt.Sprintf(
+ "no route %s on this app — this console's box serves %s",
+ host, namesOrNone(hostNames(routes), "no routes for it"))
+ }
+ if envName != "" && rt.Environment != envName {
+ if rt.Environment == "" {
+ return release.RestartScope{}, nil, fmt.Sprintf(
+ "%s is bound to no environment (it reads the app's default bundle), not %s — "+
+ "one of --host and --environment is a typo, so nothing was cycled", host, envName)
+ }
+ return release.RestartScope{}, nil, fmt.Sprintf(
+ "%s reads environment %s, not %s — one of --host and --environment is a typo, so nothing was cycled",
+ host, rt.Environment, envName)
+ }
+ return release.RestartScope{Kind: release.RestartScopeHost, Value: host}, []registry.Route{rt}, ""
+ case envName != "":
+ var hit []registry.Route
+ for _, rt := range routes {
+ if rt.Environment == envName {
+ hit = append(hit, rt)
+ }
+ }
+ if len(hit) == 0 {
+ return release.RestartScope{}, nil, fmt.Sprintf(
+ "no route of this app reads environment %s — this console's box has %s",
+ envName, namesOrNone(environmentNames(routes), "no routes for it at all"))
+ }
+ return release.RestartScope{Kind: release.RestartScopeEnvironment, Value: envName}, hit, ""
+ }
+ return release.RestartScope{Kind: release.RestartScopeApp, Value: ""}, routes, ""
+}
+
+func routeWithHost(routes []registry.Route, host string) (registry.Route, bool) {
+ for _, rt := range routes {
+ if rt.Host == host {
+ return rt, true
+ }
+ }
+ return registry.Route{}, false
+}
+
+// hostNames is every host this app has here, sorted — the "what exists" half
+// of a --host refusal.
+func hostNames(routes []registry.Route) []string {
+ out := make([]string, 0, len(routes))
+ for _, rt := range routes {
+ out = append(out, rt.Host)
+ }
+ sort.Strings(out)
+ return out
+}
+
+// environmentNames is the distinct environments this app's routes are bound
+// to, sorted, with the unbound ones named as what they are. A refusal that
+// listed only the named bundles would leave someone whose route reads the
+// default one unable to tell why their spelling "failed" — the answer being
+// that there is nothing to spell.
+func environmentNames(routes []registry.Route) []string {
+ seen := map[string]bool{}
+ var out []string
+ for _, rt := range routes {
+ name := rt.Environment
+ if name == "" {
+ // Named as what it is rather than printed as an empty string: a
+ // route with no binding reads the app's default bundle, which is
+ // an answer, not missing data. It is also not a value --environment
+ // will accept, and the parentheses are what say so.
+ name = "(routes on the default bundle, which --environment cannot name)"
+ }
+ if seen[name] {
+ continue
+ }
+ seen[name] = true
+ out = append(out, name)
+ }
+ sort.Strings(out)
+ return out
+}
+
+// namesOrNone renders a bounded list for a refusal. The cap keeps a
+// nineteen-route app readable and a two-hundred-route one from burying the
+// point; the remainder is counted rather than dropped silently.
+func namesOrNone(names []string, empty string) string {
+ if len(names) == 0 {
+ return empty
+ }
+ if len(names) <= restartScopeNamesShown {
+ return strings.Join(names, ", ")
+ }
+ return fmt.Sprintf("%s and %d more", strings.Join(names[:restartScopeNamesShown], ", "), len(names)-restartScopeNamesShown)
+}
+
+// restartRouteRows renders the addressed set, sorted by host so a caller's
+// output — and any test pinning it — is stable across passes, where registry
+// order is not.
+func restartRouteRows(routes []registry.Route) []apiRestartRoute {
+ out := make([]apiRestartRoute, 0, len(routes))
+ for _, rt := range routes {
+ out = append(out, apiRestartRoute{
+ Host: rt.Host, Environment: rt.Environment, Unit: rt.Unit, Hibernate: rt.Hibernate,
+ })
+ }
+ sort.Slice(out, func(i, j int) bool { return out[i].Host < out[j].Host })
+ return out
+}
diff --git a/internal/console/apirestartscope_test.go b/internal/console/apirestartscope_test.go
new file mode 100644
index 000000000..d6f31486e
--- /dev/null
+++ b/internal/console/apirestartscope_test.go
@@ -0,0 +1,422 @@
+package console
+
+// The narrowed restart door (apirestartscope.go, discussion #16). What is
+// pinned here, in the order it would hurt if lost: a selector that names
+// nothing is REFUSED rather than widened into an app-wide restart; the request
+// that does land goes to the log old boxes never read, leaving the app-wide
+// object untouched; a dry run writes and announces nothing; and the answer
+// names the routes it addressed, which is the whole complaint the flags exist
+// to answer.
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "slices"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/svcrole"
+)
+
+func scopeURL(fx *consoleFixture, app string) string {
+ return fx.ts.URL + "/api/cli/apps/bab/" + app + "/restart/scope"
+}
+
+// scopeFixture is a console whose registry holds a small titogo: two routes on
+// the relay bundle, one on router, one bound to nothing, one asleep, plus a
+// neighbour app and another account's row to prove the tenant boundary.
+func scopeFixture(t *testing.T, code string) (*consoleFixture, string) {
+ t.Helper()
+ fx, token := restartFixture(t, code)
+ reg, err := registry.Open(fx.regPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, rt := range []registry.Route{
+ {Host: "connect.go.tito.io", Kind: "instance", Status: "ready", App: "titogo",
+ Account: "bab", Addr: "127.0.0.1:9001", Environment: "relay"},
+ {Host: "relay2.go.tito.io", Kind: "instance", Status: "ready", App: "titogo",
+ Account: "bab", Addr: "127.0.0.1:9002", Environment: "relay", Unit: "relay2.service"},
+ {Host: "platform.ie.go.tito.io", Kind: "instance", Status: "ready", App: "titogo",
+ Account: "bab", Addr: "127.0.0.1:9003", Environment: "router"},
+ {Host: "bill.go.tito.io", Kind: "instance", Status: "ready", App: "titogo",
+ Account: "bab", Addr: "127.0.0.1:9004"},
+ {Host: "sleepy.go.tito.io", Kind: "instance", Status: "ready", App: "titogo",
+ Account: "bab", Addr: "127.0.0.1:9005", Environment: "relay", Hibernate: true},
+ {Host: "neighbour.example.com", Kind: "instance", Status: "ready", App: "kass",
+ Account: "bab", Addr: "127.0.0.1:9006", Environment: "relay"},
+ {Host: "foreign.example.com", Kind: "instance", Status: "ready", App: "titogo",
+ Account: "zzzz", Addr: "127.0.0.1:9007", Environment: "relay"},
+ } {
+ if err := reg.Upsert(rt); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return fx, token
+}
+
+// scopeLog reads the narrowed request log every NEW box watches.
+func scopeLog(t *testing.T, fx *consoleFixture, app string) *release.ScopedRestartGeneration {
+ t.Helper()
+ g, found, err := release.LoadRestartScopes(context.Background(), fx.rel, release.OpsAccount, app)
+ if err != nil {
+ t.Fatalf("read narrowed request log: %v", err)
+ }
+ if !found {
+ return &release.ScopedRestartGeneration{}
+ }
+ return g
+}
+
+type scopeAnswer struct {
+ Account string `json:"account"`
+ App string `json:"app"`
+ Scope struct {
+ Kind string `json:"kind"`
+ Value string `json:"value"`
+ } `json:"scope"`
+ DryRun bool `json:"dry_run"`
+ Requested bool `json:"requested"`
+ Routes []struct {
+ Host string `json:"host"`
+ Environment string `json:"environment"`
+ Unit string `json:"unit"`
+ Hibernate bool `json:"hibernate"`
+ } `json:"routes"`
+}
+
+func postScope(t *testing.T, fx *consoleFixture, token, body string) (int, scopeAnswer, []byte) {
+ t.Helper()
+ status, out := postBody(t, scopeURL(fx, "titogo"), token, body)
+ var res scopeAnswer
+ if status == http.StatusOK {
+ if err := json.Unmarshal(out, &res); err != nil {
+ t.Fatalf("decode %s: %v", out, err)
+ }
+ }
+ return status, res, out
+}
+
+func scopeHosts(res scopeAnswer) []string {
+ out := make([]string, 0, len(res.Routes))
+ for _, r := range res.Routes {
+ out = append(out, r.Host)
+ }
+ return out
+}
+
+// TestAPIRestartScopeEnvironmentAddressesOnlyThatBundle is the fix: a request
+// for one environment names the routes reading it — this app's, this account's
+// — and nothing else. The neighbouring app on the same bundle name and the
+// other account's identically-bound route are both absent, and that is the
+// tenant boundary, not a coincidence of the fixture.
+func TestAPIRestartScopeEnvironmentAddressesOnlyThatBundle(t *testing.T) {
+ fx, token := scopeFixture(t, "code-scope-env")
+ status, res, out := postScope(t, fx, token, `{"environment":"relay"}`)
+ if status != http.StatusOK {
+ t.Fatalf("restart scope = %d %s, want 200", status, out)
+ }
+ if !res.Requested || res.Scope.Kind != "environment" || res.Scope.Value != "relay" {
+ t.Fatalf("answer = %+v, want a requested environment scope", res)
+ }
+ want := []string{"connect.go.tito.io", "relay2.go.tito.io", "sleepy.go.tito.io"}
+ if got := scopeHosts(res); !slices.Equal(got, want) {
+ t.Fatalf("addressed %v, want %v", got, want)
+ }
+ // The sleeping route is NAMED and flagged rather than filtered out: an
+ // operator has to be able to see that it is in the set and will not cycle
+ // now, which is correct rather than a failure.
+ var sleepySeen bool
+ for _, r := range res.Routes {
+ if r.Host == "sleepy.go.tito.io" {
+ sleepySeen = r.Hibernate
+ }
+ if r.Host == "relay2.go.tito.io" && r.Unit != "relay2.service" {
+ t.Fatalf("route %+v lost the unit that owns its process", r)
+ }
+ }
+ if !sleepySeen {
+ t.Fatal("the hibernating route came back unflagged — an operator would read it as one that just cycled")
+ }
+ g := scopeLog(t, fx, "titogo")
+ if len(g.Requests) != 1 || g.Requests[0].Scope.Value != "relay" {
+ t.Fatalf("request log = %+v, want the one narrowed request", g.Requests)
+ }
+ if got := serial(t, fx); got != 1 {
+ t.Fatalf("serial after a narrowed request = %d, want 1 (otherwise boxes wait out the hourly floor)", got)
+ }
+}
+
+// TestAPIRestartScopeLeavesTheAppWideObjectAlone is the additive property at
+// the door: a narrowed request must not move the object every OLD box watches,
+// or the fleet's un-upgraded boxes would each cycle the whole app.
+func TestAPIRestartScopeLeavesTheAppWideObjectAlone(t *testing.T) {
+ fx, token := scopeFixture(t, "code-scope-additive")
+ if status, _, out := postScope(t, fx, token, `{"host":"bill.go.tito.io"}`); status != http.StatusOK {
+ t.Fatalf("restart scope = %d %s, want 200", status, out)
+ }
+ if got := restartGenEtag(t, fx, "titogo"); got != "" {
+ t.Fatalf("the app-wide restart generation exists (%q) after a narrowed request — every old box would cycle everything", got)
+ }
+}
+
+// TestAPIRestartScopeRefusesAnUnknownEnvironment: the refusal that keeps a
+// typo from becoming the bug being fixed. It names what the app actually has,
+// because a refusal exists to let somebody spot their own mistake.
+func TestAPIRestartScopeRefusesAnUnknownEnvironment(t *testing.T) {
+ fx, token := scopeFixture(t, "code-scope-typo")
+ status, _, out := postScope(t, fx, token, `{"environment":"relayy"}`)
+ if status != http.StatusBadRequest {
+ t.Fatalf("a typo'd environment = %d %s, want 400", status, out)
+ }
+ body := string(out)
+ for _, want := range []string{"relayy", "relay", "router"} {
+ if !strings.Contains(body, want) {
+ t.Fatalf("refusal %s does not name %q — it must say what it looked for and what exists", body, want)
+ }
+ }
+ if len(scopeLog(t, fx, "titogo").Requests) != 0 || restartGenEtag(t, fx, "titogo") != "" {
+ t.Fatal("a refused selector still wrote a request")
+ }
+ if got := serial(t, fx); got != 0 {
+ t.Fatalf("serial = %d after a refusal, want it untouched", got)
+ }
+}
+
+// TestAPIRestartScopeRefusesAnUnknownHost: the same rule for the other flag.
+func TestAPIRestartScopeRefusesAnUnknownHost(t *testing.T) {
+ fx, token := scopeFixture(t, "code-scope-host-typo")
+ status, _, out := postScope(t, fx, token, `{"host":"bil.go.tito.io"}`)
+ if status != http.StatusBadRequest {
+ t.Fatalf("an unknown host = %d %s, want 400", status, out)
+ }
+ if !strings.Contains(string(out), "bill.go.tito.io") {
+ t.Fatalf("refusal %s does not list the hosts this app has", out)
+ }
+ // Another account's route is not a host this caller may name, and the
+ // refusal must not leak that it exists.
+ if strings.Contains(string(out), "foreign.example.com") {
+ t.Fatalf("refusal %s named another account's route", out)
+ }
+}
+
+// TestAPIRestartScopeRefusesAMismatchedPair is the precedence rule's teeth.
+// --host wins over --environment, but only after the host is checked against
+// it: quietly ignoring the wider flag would hide that one of the two is a
+// typo, and this is not a verb to guess on.
+func TestAPIRestartScopeRefusesAMismatchedPair(t *testing.T) {
+ fx, token := scopeFixture(t, "code-scope-mismatch")
+ status, _, out := postScope(t, fx, token, `{"host":"platform.ie.go.tito.io","environment":"relay"}`)
+ if status != http.StatusBadRequest {
+ t.Fatalf("a mismatched pair = %d %s, want 400", status, out)
+ }
+ if !strings.Contains(string(out), "router") || !strings.Contains(string(out), "relay") {
+ t.Fatalf("refusal %s must name what the host actually reads and what was asked for", out)
+ }
+ if len(scopeLog(t, fx, "titogo").Requests) != 0 {
+ t.Fatal("a mismatched pair still wrote a request")
+ }
+}
+
+// TestAPIRestartScopeHostWinsOverAMatchingEnvironment: the pair is accepted
+// when it agrees, and the narrower flag is what lands.
+func TestAPIRestartScopeHostWinsOverAMatchingEnvironment(t *testing.T) {
+ fx, token := scopeFixture(t, "code-scope-pair")
+ status, res, out := postScope(t, fx, token, `{"host":"connect.go.tito.io","environment":"relay"}`)
+ if status != http.StatusOK {
+ t.Fatalf("a matching pair = %d %s, want 200", status, out)
+ }
+ if res.Scope.Kind != "host" || res.Scope.Value != "connect.go.tito.io" {
+ t.Fatalf("scope = %+v, want the host to win", res.Scope)
+ }
+ if got := scopeHosts(res); !slices.Equal(got, []string{"connect.go.tito.io"}) {
+ t.Fatalf("addressed %v, want the one route", got)
+ }
+}
+
+// TestAPIRestartScopeDryRunTouchesNothing: the answer an operator wanted
+// before cycling anything — the set, and no restart.
+func TestAPIRestartScopeDryRunTouchesNothing(t *testing.T) {
+ fx, token := scopeFixture(t, "code-scope-dry")
+ status, res, out := postScope(t, fx, token, `{"environment":"relay","dry_run":true}`)
+ if status != http.StatusOK {
+ t.Fatalf("dry run = %d %s, want 200", status, out)
+ }
+ if res.Requested || !res.DryRun {
+ t.Fatalf("answer = %+v, want dry_run true and requested false", res)
+ }
+ if got := scopeHosts(res); len(got) != 3 {
+ t.Fatalf("dry run addressed %v, want the same three routes the real request would", got)
+ }
+ if len(scopeLog(t, fx, "titogo").Requests) != 0 || restartGenEtag(t, fx, "titogo") != "" {
+ t.Fatal("a dry run wrote a request")
+ }
+ if got := serial(t, fx); got != 0 {
+ t.Fatalf("serial = %d after a dry run, want it untouched — a dry run must not even wake the fleet", got)
+ }
+}
+
+// TestAPIRestartScopeDryRunOfTheWholeApp: with no selector, the dry run is how
+// somebody sees what a plain `carlos restart --app` would cycle. That is the
+// blast radius the reporter said they would not run the verb to discover.
+func TestAPIRestartScopeDryRunOfTheWholeApp(t *testing.T) {
+ fx, token := scopeFixture(t, "code-scope-dry-app")
+ status, res, out := postScope(t, fx, token, `{"dry_run":true}`)
+ if status != http.StatusOK {
+ t.Fatalf("app-wide dry run = %d %s, want 200", status, out)
+ }
+ if res.Scope.Kind != "app" {
+ t.Fatalf("scope = %+v, want the whole app", res.Scope)
+ }
+ want := []string{"bill.go.tito.io", "connect.go.tito.io", "platform.ie.go.tito.io", "relay2.go.tito.io", "sleepy.go.tito.io"}
+ if got := scopeHosts(res); !slices.Equal(got, want) {
+ t.Fatalf("addressed %v, want every route of the app: %v", got, want)
+ }
+}
+
+// TestAPIRestartScopeRefusesAWriteWithNoSelector is the door's own
+// never-widen rule, turned on itself.
+//
+// This path exists BECAUSE the app-wide door widens a request it cannot read.
+// A body with no selector in it — a bare `{}`, or a client that renamed a
+// field — resolves to the whole app, and performing that here would be the
+// nineteen-route accident committed through the door built to prevent it. So
+// the app-wide case is answered only as a --dry-run (which is how somebody
+// sees the blast radius before typing the real verb), and a WRITE with no
+// narrowing is refused, naming the door that does own it.
+func TestAPIRestartScopeRefusesAWriteWithNoSelector(t *testing.T) {
+ fx, token := scopeFixture(t, "code-scope-appwide")
+ status, _, out := postScope(t, fx, token, `{}`)
+ if status != http.StatusBadRequest {
+ t.Fatalf("an app-wide write through the scope door = %d %s, want 400", status, out)
+ }
+ if !strings.Contains(string(out), "carlos restart --app") {
+ t.Fatalf("refusal %s does not name the door that cycles the whole app", out)
+ }
+ if got := restartGenEtag(t, fx, "titogo"); got != "" {
+ t.Fatalf("a refused app-wide write still moved the object every box watches (%q) — that is the whole app cycled", got)
+ }
+ if len(scopeLog(t, fx, "titogo").Requests) != 0 {
+ t.Fatal("a refused app-wide write still appended to the narrowed log")
+ }
+ if got := serial(t, fx); got != 0 {
+ t.Fatalf("serial = %d after a refusal, want it untouched", got)
+ }
+ // The dry run of the same body is still answered: that is the one form of
+ // app-wide this door is for.
+ if status, res, out := postScope(t, fx, token, `{"dry_run":true}`); status != http.StatusOK || res.Requested {
+ t.Fatalf("app-wide dry run = %d %s (%+v), want 200 and nothing requested", status, out, res)
+ }
+}
+
+// TestAPIRestartScopeRefusesAFieldItDoesNotKnow is the same rule one layer
+// out. A misspelled or renamed selector key that was merely IGNORED would
+// leave a body with no selector in it, which is a whole-app restart requested
+// by a client that asked for one route. The refusal names the field, because
+// the fix is a spelling.
+func TestAPIRestartScopeRefusesAFieldItDoesNotKnow(t *testing.T) {
+ fx, token := scopeFixture(t, "code-scope-unknown-field")
+ status, _, out := postScope(t, fx, token, `{"hosts":"relay2.go.tito.io"}`)
+ if status != http.StatusBadRequest {
+ t.Fatalf(`{"hosts":...} = %d %s, want 400 — an ignored key here is an app-wide restart`, status, out)
+ }
+ if !strings.Contains(string(out), "hosts") {
+ t.Fatalf("refusal %s does not name the field, so nobody can find the typo", out)
+ }
+ if restartGenEtag(t, fx, "titogo") != "" || len(scopeLog(t, fx, "titogo").Requests) != 0 {
+ t.Fatal("a body this door refused still wrote a request")
+ }
+}
+
+// TestAPIRestartScopeGateIsTheFamilys404: an app the account never claimed, a
+// stranger's token and a credential bound to another app are ONE answer, byte
+// for byte the answer the app-wide door gives — this repo's rule for these
+// gates is byte-identical, not merely same-status, because a differently
+// shaped 404 is an oracle that says "this door exists for you, that app does
+// not".
+func TestAPIRestartScopeGateIsTheFamilys404(t *testing.T) {
+ fx, _ := scopeFixture(t, "code-scope-gate")
+ // The old door's refusal, read from the old door rather than spelled out
+ // here, so the two cannot drift apart without this failing.
+ eve := cliToken(t, fx, "eve@keymail.dev", "code-scope-gate-eve")
+ oldStatus, oldBody := postBody(t, fx.ts.URL+"/api/cli/apps/bab/titogo/restart", eve, "")
+ if oldStatus != http.StatusNotFound {
+ t.Fatalf("the app-wide door answered a stranger %d, want 404", oldStatus)
+ }
+ for _, c := range []struct {
+ name string
+ url string
+ token string
+ }{
+ {"a stranger", scopeURL(fx, "titogo"), eve},
+ // An app-scoped service credential reaching PAST its own app. The
+ // binding is checked before the app is even looked up, so "not your
+ // app" and "no such app" stay the same answer.
+ {"a credential bound to another app", scopeURL(fx, "titogo"),
+ serviceToken(t, fx.st, OpsAccountID, "kassbot", "kass", svcrole.Operate)},
+ {"an app this account never claimed", scopeURL(fx, "nosuchapp"),
+ cliToken(t, fx, "alice@keymail.dev", "code-scope-gate-alice")},
+ } {
+ status, body := postBody(t, c.url, c.token, `{"environment":"relay"}`)
+ if status != oldStatus || string(body) != string(oldBody) {
+ t.Fatalf("%s got %d %q, want the app-wide door's own %d %q",
+ c.name, status, body, oldStatus, oldBody)
+ }
+ }
+ if len(scopeLog(t, fx, "titogo").Requests) != 0 || restartGenEtag(t, fx, "titogo") != "" {
+ t.Fatal("a refused caller still wrote a request")
+ }
+ if got := serial(t, fx); got != 0 {
+ t.Fatalf("serial = %d after refusals, want it untouched", got)
+ }
+}
+
+// TestAPIRestartScopeServiceCredentialReachesItsOwnApp is the other half of
+// the case above: the 404 there is about the app binding, not about services
+// being turned away from this door. An Operate credential bound to titogo
+// narrows a restart of titogo exactly as a member does.
+func TestAPIRestartScopeServiceCredentialReachesItsOwnApp(t *testing.T) {
+ fx, _ := scopeFixture(t, "code-scope-svc-own")
+ tok := serviceToken(t, fx.st, OpsAccountID, "titogobot", "titogo", svcrole.Operate)
+ status, res, out := postScope(t, fx, tok, `{"host":"bill.go.tito.io"}`)
+ if status != http.StatusOK {
+ t.Fatalf("a credential bound to titogo = %d %s, want 200", status, out)
+ }
+ if !res.Requested || res.Scope.Kind != "host" || res.Scope.Value != "bill.go.tito.io" {
+ t.Fatalf("answer = %+v, want the narrowed request", res)
+ }
+}
+
+// TestAPIRestartScopeNeedsARegistry: a console that cannot see routes cannot
+// resolve a selector or report a set, and must not answer "requested" for
+// something it never checked. The app-wide door still works there, and the
+// refusal says so.
+func TestAPIRestartScopeNeedsARegistry(t *testing.T) {
+ fx, token := scopeFixture(t, "code-scope-noreg")
+ fx.srv.cfg.Registry = ""
+ status, _, out := postScope(t, fx, token, `{"environment":"relay"}`)
+ if status != http.StatusServiceUnavailable {
+ t.Fatalf("a registry-less console answered %d %s, want 503", status, out)
+ }
+ if !strings.Contains(string(out), "carlos restart --app") {
+ t.Fatalf("refusal %s does not name the form that still works", out)
+ }
+}
+
+// TestAPIRestartScopeRefusesAMalformedSelector keeps a hostile value out
+// before the registry is even opened.
+func TestAPIRestartScopeRefusesAMalformedSelector(t *testing.T) {
+ fx, token := scopeFixture(t, "code-scope-malformed")
+ for _, body := range []string{
+ `{"host":"../../etc/passwd"}`,
+ `{"environment":"NOT VALID"}`,
+ `{"environment":"default"}`,
+ } {
+ if status, _, out := postScope(t, fx, token, body); status != http.StatusBadRequest {
+ t.Fatalf("%s = %d %s, want 400", body, status, out)
+ }
+ }
+}
diff --git a/internal/console/authcli.go b/internal/console/authcli.go
index 9875f2b64..c44a82887 100644
--- a/internal/console/authcli.go
+++ b/internal/console/authcli.go
@@ -489,6 +489,13 @@ func (s *Server) registerCLIRoutes() {
// promote and rollback, and the only one whose subject is a running
// process rather than a pointer.
s.cliRoute("POST /api/cli/apps/{account}/{app}/restart", svcrole.Allow(svcrole.Operate), s.handleAPIRestart)
+ // The NARROWED form (discussion #16): one route, or everything reading one
+ // config environment, plus the dry run that shows the set first. A
+ // SEPARATE path rather than a body on the door above, and deliberately so:
+ // that one reads no body, so a console predating this would answer a
+ // narrowed request by cycling the whole app. Here it answers 404 and the
+ // CLI says why. Same role — the subject is still a running process.
+ s.cliRoute("POST /api/cli/apps/{account}/{app}/restart/scope", svcrole.Allow(svcrole.Operate), s.handleAPIRestartScope)
// `carlos schedule` (apischedule.go, scheduled-work spec §8): declare
// recurring work, take it back, and make it fire now. The RESTART class,
// not the publish one — operating a running app rather than moving a
@@ -822,18 +829,51 @@ func readJSON(w http.ResponseWriter, r *http.Request, out any) bool {
// its own 413 rather than hiding inside "malformed JSON body" — the caller's
// fix (split the request, shorten the value) is nothing like a JSON fix.
func readJSONLimit(w http.ResponseWriter, r *http.Request, out any, limit int64) bool {
+ return decodeJSONBody(w, r, out, limit, false)
+}
+
+// readJSONStrictLimit is readJSONLimit that also refuses a field the struct
+// does not declare, with a 400 that NAMES it.
+//
+// It is for the doors where an unrecognised key would otherwise be dropped and
+// the request acted on as if it had not been sent — which, at a door whose job
+// is to NARROW something, silently widens it. The narrowed restart door
+// (apirestartscope.go) is the case it was written for: a client that renames
+// `host` to `hosts` would, without this, POST a body with no selector in it
+// and be answered by a restart of every route the app owns. A door that exists
+// because another door widens must not itself widen on a typo.
+//
+// The unknown-field check is a prefix match on encoding/json's own message
+// because the package gives no typed error for it; the message is stable and
+// already names the field, which is the whole thing the caller needs.
+func readJSONStrictLimit(w http.ResponseWriter, r *http.Request, out any, limit int64) bool {
+ return decodeJSONBody(w, r, out, limit, true)
+}
+
+// decodeJSONBody is the shared body of the two above: content type, byte cap,
+// decode, and one refusal per failure shape.
+func decodeJSONBody(w http.ResponseWriter, r *http.Request, out any, limit int64, strict bool) bool {
ct, _, _ := strings.Cut(r.Header.Get("Content-Type"), ";")
if strings.TrimSpace(strings.ToLower(ct)) != "application/json" {
jsonError(w, http.StatusUnsupportedMediaType, "send a JSON body with Content-Type: application/json")
return false
}
r.Body = http.MaxBytesReader(w, r.Body, limit)
- if err := json.NewDecoder(r.Body).Decode(out); err != nil {
+ dec := json.NewDecoder(r.Body)
+ if strict {
+ dec.DisallowUnknownFields()
+ }
+ if err := dec.Decode(out); err != nil {
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
jsonError(w, http.StatusRequestEntityTooLarge, "request body too large")
return false
}
+ if strict && strings.HasPrefix(err.Error(), "json: unknown field ") {
+ jsonError(w, http.StatusBadRequest, err.Error()+
+ " — this door refuses a body it does not fully understand rather than acting on the part it does")
+ return false
+ }
jsonError(w, http.StatusBadRequest, "malformed JSON body")
return false
}
diff --git a/internal/host/agent.go b/internal/host/agent.go
index d2da8db09..130b6e1ce 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -1436,12 +1436,14 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu
// Built once, used by both pass shapes below — see fastPassOpts for the
// DO-NOT list and why the stage requester is per-loop, not per-tick.
opts := fastPassOpts(dd, stageRequester(dd), restart, makeStartRoute(reg, systemctlStartUnit))
- // The restart check's memory (issue #113), seeded ONCE here — one read per
- // app at arming and never again except on a serial move, which is what
- // keeps an idle box at exactly one serial GET per tick. See restartSeed for
- // why the baseline cannot wait for the first request to establish itself.
- restartCache := map[[2]string]string{}
- restartSeed(ctx, reg, store, dd, restartCache)
+ // The restart check's memory (issue #113, narrowed requests added by
+ // discussion #16), seeded ONCE here — two reads per app at arming (the
+ // app-wide generation and the narrowed request log) and never again except
+ // on a serial move, which is what keeps an idle box at exactly one serial
+ // GET per tick. See restartSeed for why the baseline cannot wait for the
+ // first request to establish itself.
+ restartMem := newRestartMemory()
+ restartSeed(ctx, reg, store, dd, restartMem)
restartBounce := bounceFn(svc)
// The schedule converge's own seed, on restartSeed's precedent and for the
// same shape of reason: the leg below runs only on a serial MOVE, so
@@ -1496,7 +1498,7 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu
log.Printf("agent: restart check: list routes: %v", err)
return
}
- restartRequestLeg(ctx, routes, store, dd, restartBounce, restartCache)
+ restartRequestLeg(ctx, routes, store, dd, restartBounce, restartMem)
}
// The schedule leg: converge this box's timer rows from the apps'
// declarations. Nil when the box has no scheduled work armed, which
diff --git a/internal/host/restart.go b/internal/host/restart.go
index cfd75554e..757f7b740 100644
--- a/internal/host/restart.go
+++ b/internal/host/restart.go
@@ -188,7 +188,7 @@ func bounceExecRoutes(routes []registry.Route, moved [][2]string, bounce func(ho
}
n := 0
for _, rt := range routes {
- if !release.Adoptable(rt) || rt.Unit != "" || rt.Backing == registry.BackingUnit {
+ if !release.Adoptable(rt) || !execBounceable(rt) {
continue
}
if !want[[2]string{release.RouteAccount(rt), rt.App}] {
@@ -266,6 +266,25 @@ func restartMarksPath(dd string) string { return filepath.Join(dd, restartMarksF
type restartMarks struct {
Schema int `json:"schema"` // 1
Apps map[string]string `json:"apps"`
+ // Scopes is the same memory for NARROWED requests (discussion #16),
+ // keyed identically: the id of the last request in
+ // release.RestartScopeKey's log this box has acted on, or
+ // release.RestartScopeNone for "looked, nothing there".
+ //
+ // Additive in both directions, because this file is read and written by
+ // two binaries that roll at different times. A binary that predates the
+ // field ignores it on read (encoding/json drops unknown keys) and DROPS
+ // it on write — which costs a downgraded box exactly one re-baseline,
+ // after which it cycles nothing it was not asked to.
+ //
+ // The other direction is free of REPLAYS but not of drops, and the
+ // difference matters: a marks file with no `scopes` key is a box that has
+ // never looked at the log, which is a first sight, which is a seed — safe,
+ // and exactly wrong for a narrowed request that lands before that seed is
+ // written. Since EVERY box that ran the app-wide feature has such a file,
+ // that window is closed from the agent's side rather than left to the
+ // hourly floor; see seedRestartMarks' needRootPass.
+ Scopes map[string]string `json:"scopes,omitempty"`
}
// restartAbsent is the marks value for an app this box has SEEN and found to
@@ -316,6 +335,13 @@ func lockRestartMarks(path string) (unlock func(), err error) {
}, nil
}
+// newRestartMarks is an empty memory: both maps non-nil, so every caller can
+// write to them without a nil check and a marks file that predates one of them
+// cannot leave a nil map behind.
+func newRestartMarks() *restartMarks {
+ return &restartMarks{Schema: 1, Apps: map[string]string{}, Scopes: map[string]string{}}
+}
+
// loadRestartMarks reads the marks file. `known` is the seed-don't-replay
// switch and the only subtle thing here: it is FALSE when the file is missing,
// unreadable, or does not parse — three different accidents with one correct
@@ -330,7 +356,7 @@ func lockRestartMarks(path string) (unlock func(), err error) {
// other way is a box that cycles every app it serves the moment its state file
// goes missing.
func loadRestartMarks(path string) (marks *restartMarks, known bool) {
- m := &restartMarks{Schema: 1, Apps: map[string]string{}}
+ m := newRestartMarks()
raw, err := os.ReadFile(path)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
@@ -340,11 +366,19 @@ func loadRestartMarks(path string) (marks *restartMarks, known bool) {
}
if err := json.Unmarshal(raw, m); err != nil {
log.Printf("adopt: restart marks %s do not parse (%v) — re-baselining, no restarts this pass", path, err)
- return &restartMarks{Schema: 1, Apps: map[string]string{}}, false
+ return newRestartMarks(), false
}
if m.Apps == nil {
m.Apps = map[string]string{}
}
+ // A file written before narrowed restarts existed carries no `scopes` key
+ // at all; that is a first sight of every app's log, not an empty memory of
+ // one, and the seed rules below turn it into exactly that. Which is why
+ // the agent asks for a pass that performs that seed as it arms, instead of
+ // leaving a request typed in the meantime to meet it — scopeBaselineIncomplete.
+ if m.Scopes == nil {
+ m.Scopes = map[string]string{}
+ }
m.Schema = 1
return m, true
}
@@ -413,36 +447,106 @@ func (m *restartMarks) save(path string) error {
// Absent objects ARE recorded, with restartAbsent: "looked, nothing there" is
// the fact that makes an app's first ever request movement rather than a first
// sight.
-func seedRestartMarks(path string, cache map[[2]string]string) error {
- if len(cache) == 0 {
- return nil // nothing was seen; nothing may be claimed
+//
+// needRootPass is the SECOND half of the handover, and it exists because
+// create-only cannot repair a baseline that is merely INCOMPLETE. Every box
+// that ran the app-wide feature already has a marks file, written before
+// narrowed restarts existed and so carrying no `scopes` key at all. This
+// function will not touch it — that is the whole create-only rule — so the
+// narrowed mark stays missing until a ROOT pass records it, and until then
+// restartScopeLeg reads every app as a first sight and cycles nothing for a
+// request the CLI has already answered "restart requested". Exec-shaped routes
+// survive that (the agent bounces its own children from mem.scope), but
+// unit-shaped ones — Tito's relay and router, the shape these flags exist for
+// — are dropped silently, for as long as it takes the next root pass to run:
+// worst case the hourly timer.
+//
+// So when the baseline on disk is missing a narrowed mark for an app this box
+// serves, say so, and let restartSeed ask for ONE root pass. That pass is the
+// same pass carlos-adopt.timer runs anyway, under the same lock, against the
+// same marks file and the same seed rules — this changes WHEN it runs, never
+// what it decides — so it can neither replay a request (it dispatches on the
+// per-app marks, which nothing here writes) nor drop one (it strictly shrinks
+// the window in which a narrowed request meets a pass with no narrowed
+// baseline, from an hour to the moment the agent arms, which is the same floor
+// seedRestartMarks already buys the app-wide half).
+func seedRestartMarks(path string, mem *restartMemory) (needRootPass bool, err error) {
+ if mem == nil || len(mem.gen) == 0 {
+ return false, nil // nothing was seen; nothing may be claimed
}
if _, err := os.Stat(path); err == nil {
- return nil // the root pass (or an earlier agent) already has a baseline
+ // The root pass (or an earlier agent) already has a baseline — but it
+ // may predate narrowing, and a half-baseline is what silently eats a
+ // narrowed request.
+ return scopeBaselineIncomplete(path, mem), nil
} else if !errors.Is(err, os.ErrNotExist) {
- return err
+ return false, err
}
- m := &restartMarks{Schema: 1, Apps: map[string]string{}}
- for k, etag := range cache {
+ m := newRestartMarks()
+ for k, etag := range mem.gen {
m.Apps[restartMarkKey(k[0], k[1])] = etag
}
+ // Both memories or neither: restartSeed only gets here when every app's
+ // BOTH objects read cleanly, so a baseline that recorded one and left the
+ // other for the root pass to establish would be a baseline with a known
+ // hole in it — which is the one shape restartPass refuses to write.
+ for k, id := range mem.scope {
+ m.Scopes[restartMarkKey(k[0], k[1])] = id
+ }
raw, err := json.MarshalIndent(m, "", " ")
if err != nil {
- return err
+ return false, err
}
raw = append(raw, '\n')
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
if errors.Is(err, os.ErrExist) {
- return nil // lost the race to the root pass; its mark is the better one
+ // Lost the race to the root pass; its mark is the better one — but
+ // a pass running an OLD binary writes no scopes key, so the same
+ // question has to be asked of what actually landed.
+ return scopeBaselineIncomplete(path, mem), nil
}
- return err
+ return false, err
}
_, writeErr := f.Write(raw)
if err := f.Close(); err != nil && writeErr == nil {
writeErr = err
}
- return writeErr
+ // The file this call just wrote carries both memories, so there is nothing
+ // left for a root pass to record.
+ return false, writeErr
+}
+
+// scopeBaselineIncomplete reports whether the marks file on disk leaves the
+// ROOT pass with no narrowed-restart position for an app this box serves.
+//
+// Two shapes count, and they are the same fault:
+//
+// - the file does not load (missing, unreadable, unparseable), so the next
+// pass re-baselines everything — which is right, and worth doing NOW
+// rather than at the hourly floor, since a re-baselining pass dispatches
+// nothing;
+// - the file loads but has no `scopes` entry for some app in this sweep,
+// which is exactly what a marks file written before narrowing existed
+// looks like.
+//
+// It reads and never writes. The agent cannot take restartPass's lock — that
+// lock file is created by root and doubles as the ROOT pass's writability
+// test — and a lock-free read-modify-write here would race a running pass and
+// revert the advance it had just made, which is the one accident the lock
+// exists to prevent. Reporting the gap and letting the pass that owns the file
+// fill it in is the whole fix.
+func scopeBaselineIncomplete(path string, mem *restartMemory) bool {
+ marks, known := loadRestartMarks(path)
+ if !known {
+ return true
+ }
+ for k := range mem.scope {
+ if _, seen := marks.Scopes[restartMarkKey(k[0], k[1])]; !seen {
+ return true
+ }
+ }
+ return false
}
// restartPass is the ROOT half: for every app on this box whose restart
@@ -514,6 +618,10 @@ func restartPass(ctx context.Context, routes []registry.Route, store release.Sto
marks, known := loadRestartMarks(marksPath)
changed := false
readFailed := false
+ // What this pass has already cycled. A route restarted by an app-wide
+ // request must not be restarted a second time by a narrowed one that
+ // landed in the same window: one pass, one cycle per host.
+ already := map[string]bool{}
for _, k := range apps {
_, etag, err := store.Get(ctx, release.RestartGenKey(k[0], k[1]))
switch {
@@ -554,6 +662,7 @@ func restartPass(ctx context.Context, routes []registry.Route, store release.Sto
log.Printf("adopt: restart requested for %s/%s: %s did not cycle: %v", k[0], k[1], rt.Host, err)
continue
}
+ already[rt.Host] = true
dispatched++
}
// One line per APP, not per route, and it says DISPATCHED rather than
@@ -566,6 +675,12 @@ func restartPass(ctx context.Context, routes []registry.Route, store release.Sto
marks.Apps[key] = etag
changed = true
}
+ // The narrowed half (discussion #16), under the SAME lock, against the
+ // same marks file and the same seed rules — a second pass with a second
+ // lock would be two memories of one box's restarts, free to disagree.
+ scopeChanged, scopeReadFailed := restartScopeLeg(ctx, routes, store, apps, marks, known, restart, already)
+ changed = changed || scopeChanged
+ readFailed = readFailed || scopeReadFailed
if !known && readFailed {
// A baseline that omits the apps a blip hid is worse than no baseline:
// this pass would write "known", and the next one would read those
@@ -599,22 +714,30 @@ func restartPass(ctx context.Context, routes []registry.Route, store release.Sto
// instant, while the marker write can block on a busy /data, and the children
// bounced here are the ones cycled here or nowhere.
func restartRequestLeg(ctx context.Context, routes []registry.Route, store release.Store, dd string,
- bounce func(host string) bool, cache map[[2]string]string) bool {
- moved, readErrs := restartGenPass(ctx, routes, store, cache)
+ bounce func(host string) bool, mem *restartMemory) bool {
+ moved, readErrs := restartGenPass(ctx, routes, store, mem.gen)
+ // The narrowed requests are read on the same move and reported the same
+ // way. Both, always: an app-wide request and a narrowed one can land in
+ // one poll window, and a leg that stopped at the first would leave the
+ // other for the hourly floor.
+ scoped, scopeErrs := restartScopePass(ctx, routes, store, mem.scope)
+ readErrs = append(readErrs, scopeErrs...)
if len(readErrs) > 0 {
// ONE line per pass, never one per app: the count carries the scale,
// the first error the cause — they are the same cause in practice.
log.Printf("agent: restart check: %d apps unreadable: %v", len(readErrs), readErrs[0])
}
- if len(moved) == 0 {
+ if len(moved) == 0 && len(scoped) == 0 {
return false
}
bounceExecRoutes(routes, moved, bounce)
+ bounceScopedRoutes(routes, scoped, bounce)
if err := requestRootRestart(dd); err != nil {
log.Printf("agent: request root restart pass: %v", err)
return false
}
- log.Printf("agent: restart requested for %d app(s) on this box, asked for a root restart pass", len(moved))
+ log.Printf("agent: restart requested for %d app(s) and narrowed for %d on this box, asked for a root restart pass",
+ len(moved), len(scoped))
return true
}
@@ -633,18 +756,41 @@ func restartRequestLeg(ctx context.Context, routes []registry.Route, store relea
// sight — safe) and blocks the marks seed entirely, for the reason
// seedRestartMarks and restartPass both give: a baseline with holes in it is
// the thing that manufactures restarts.
-func restartSeed(ctx context.Context, reg *registry.Registry, store release.Store, dd string, cache map[[2]string]string) {
+//
+// The one thing it may ask for is a root pass, and only when the baseline on
+// disk is INCOMPLETE — see seedRestartMarks' needRootPass. That is the case of
+// every box already carrying a pre-narrowing marks file, and asking is what
+// keeps the first narrowed request after a roll from being answered "restart
+// requested" and then cycling nothing unit-shaped. It is one marker touch,
+// once, at arming: the pass it starts is the ordinary one, and once that pass
+// has recorded the narrowed marks no later agent start asks again.
+func restartSeed(ctx context.Context, reg *registry.Registry, store release.Store, dd string, mem *restartMemory) {
routes, err := reg.List()
if err != nil {
log.Printf("agent: restart baseline: list routes: %v (the first request may need a second try)", err)
return
}
- _, readErrs := restartGenPass(ctx, routes, store, cache)
+ _, readErrs := restartGenPass(ctx, routes, store, mem.gen)
+ _, scopeErrs := restartScopePass(ctx, routes, store, mem.scope)
+ readErrs = append(readErrs, scopeErrs...)
if len(readErrs) > 0 {
log.Printf("agent: restart baseline: %d apps unreadable: %v", len(readErrs), readErrs[0])
return
}
- if err := seedRestartMarks(restartMarksPath(dd), cache); err != nil {
+ needRootPass, err := seedRestartMarks(restartMarksPath(dd), mem)
+ if err != nil {
log.Printf("agent: seed restart marks: %v", err)
+ return
+ }
+ if !needRootPass {
+ return
+ }
+ if err := requestRootRestart(dd); err != nil {
+ // Latency, not correctness: carlos-adopt.timer's hourly pass records
+ // the same marks, it just leaves the window open for longer.
+ log.Printf("agent: ask for a baselining root restart pass: %v", err)
+ return
}
+ log.Printf("agent: restart baseline: the root pass has no narrowed-restart mark for at least one app on this box " +
+ "(a marks file written before narrowed restarts existed) — asked for one root pass to record it")
}
diff --git a/internal/host/restart_test.go b/internal/host/restart_test.go
index bf59e7a98..bf302619a 100644
--- a/internal/host/restart_test.go
+++ b/internal/host/restart_test.go
@@ -260,11 +260,11 @@ func TestRestartRequestLegBouncesAndRelays(t *testing.T) {
}
var bounced []string
bounce := func(h string) bool { bounced = append(bounced, h); return true }
- cache := map[[2]string]string{}
+ mem := newRestartMemory()
// Baseline (what restartSeed does at arming): silent, no marker.
askRestart(t, ctx, s, "bab", "eleven")
- if asked := restartRequestLeg(ctx, routes, s, dd, bounce, cache); asked {
+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, mem); asked {
t.Fatal("the baselining leg asked for a root pass — first sight must be silent")
}
if _, err := os.Stat(marker); !errors.Is(err, os.ErrNotExist) {
@@ -272,7 +272,7 @@ func TestRestartRequestLegBouncesAndRelays(t *testing.T) {
}
askRestart(t, ctx, s, "bab", "eleven")
- if asked := restartRequestLeg(ctx, routes, s, dd, bounce, cache); !asked {
+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, mem); !asked {
t.Fatal("a moved restart generation must ask for a root pass")
}
if !slices.Equal(bounced, []string{"exec.example.com"}) {
@@ -284,7 +284,7 @@ func TestRestartRequestLegBouncesAndRelays(t *testing.T) {
// And a leg with nothing moved neither bounces nor re-asks.
bounced = nil
- if asked := restartRequestLeg(ctx, routes, s, dd, bounce, cache); asked {
+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, mem); asked {
t.Fatal("an unmoved generation asked for a root pass")
}
if len(bounced) != 0 {
@@ -639,22 +639,40 @@ func TestRestartPassUnreadableMarksReBaseline(t *testing.T) {
// sentinel is what makes an app's first ever request movement.
func TestSeedRestartMarksIsCreateOnly(t *testing.T) {
path := filepath.Join(t.TempDir(), "restart-marks.json")
- cache := map[[2]string]string{
+ mem := newRestartMemory()
+ mem.gen = map[[2]string]string{
{"bab", "eleven"}: "etag-1",
{"bab", "titogo"}: genAbsent, // seen, and it has no object
}
- if err := seedRestartMarks(path, cache); err != nil {
+ mem.scope = map[[2]string]string{
+ {"bab", "eleven"}: release.RestartScopeNone,
+ }
+ need, err := seedRestartMarks(path, mem)
+ if err != nil {
t.Fatalf("seed: %v", err)
}
+ if need {
+ // The file this seed just wrote carries both memories, so there is
+ // nothing for a root pass to fill in.
+ t.Fatal("a seed that wrote the whole baseline still asked for a root pass")
+ }
m := marksOf(t, path)
if m.Apps[restartMarkKey("bab", "eleven")] != "etag-1" ||
m.Apps[restartMarkKey("bab", "titogo")] != restartAbsent {
t.Fatalf("seeded marks = %v, want the etag and the recorded absence", m.Apps)
}
+ // The narrowed memory is seeded by the same sweep and in the same file:
+ // a baseline that recorded one signal and not the other would leave the
+ // root pass reading the second one as a first sight for a second time.
+ if m.Scopes[restartMarkKey("bab", "eleven")] != release.RestartScopeNone {
+ t.Fatalf("seeded scope marks = %v, want the recorded empty log", m.Scopes)
+ }
- cache[[2]string{"bab", "eleven"}] = "etag-2"
- if err := seedRestartMarks(path, cache); err != nil {
+ mem.gen[[2]string{"bab", "eleven"}] = "etag-2"
+ if need, err := seedRestartMarks(path, mem); err != nil {
t.Fatalf("second seed: %v", err)
+ } else if need {
+ t.Fatal("a complete baseline on disk still asked for a root pass — every agent start would ask")
}
if got := marksOf(t, path).Apps[restartMarkKey("bab", "eleven")]; got != "etag-1" {
t.Fatalf("mark = %q after a second seed, want the original %q — seeding must never overwrite", got, "etag-1")
@@ -667,8 +685,10 @@ func TestSeedRestartMarksIsCreateOnly(t *testing.T) {
// the box later gains as movement.
func TestSeedRestartMarksRefusesAnEmptyBaseline(t *testing.T) {
path := filepath.Join(t.TempDir(), "restart-marks.json")
- if err := seedRestartMarks(path, map[[2]string]string{}); err != nil {
+ if need, err := seedRestartMarks(path, newRestartMemory()); err != nil {
t.Fatalf("seed: %v", err)
+ } else if need {
+ t.Fatal("a sweep that saw nothing asked for a root pass; it may claim nothing at all")
}
if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("an empty sweep wrote %s (err %v), want no file at all", path, err)
@@ -687,14 +707,15 @@ func TestSeededBaselineIsWhatTheRootPassTrusts(t *testing.T) {
askRestart(t, ctx, s, "bab", "eleven") // an older request, already served
// The agent's first tick: baseline in memory, baseline on disk.
- cache := map[[2]string]string{}
- restartGenPass(ctx, routes, s, cache)
- if err := seedRestartMarks(restartMarksPath(dd), cache); err != nil {
+ mem := newRestartMemory()
+ restartGenPass(ctx, routes, s, mem.gen)
+ restartScopePass(ctx, routes, s, mem.scope)
+ if _, err := seedRestartMarks(restartMarksPath(dd), mem); err != nil {
t.Fatalf("seed: %v", err)
}
askRestart(t, ctx, s, "bab", "eleven") // the member types `carlos restart`
- moved, _ := restartGenPass(ctx, routes, s, cache)
+ moved, _ := restartGenPass(ctx, routes, s, mem.gen)
if len(moved) != 1 {
t.Fatalf("the agent missed the request: moved = %v", moved)
}
diff --git a/internal/host/restartscope.go b/internal/host/restartscope.go
new file mode 100644
index 000000000..55f1d3ba9
--- /dev/null
+++ b/internal/host/restartscope.go
@@ -0,0 +1,289 @@
+// The box half of a NARROWED restart (discussion #16): the same two-process
+// dance restart.go describes, reading the request log at
+// release.RestartScopeKey instead of the app-wide generation, and cycling only
+// the routes a request actually names.
+//
+// Everything load-bearing in restart.go still holds here, and this file is
+// deliberately built to inherit rather than restate it:
+//
+// - It rides the SERIAL. No poll of its own; the agent's existing 2s serial
+// read is what makes either object get looked at, and an idle box reads
+// neither. The cost of this feature at rest is zero, and on a serial MOVE
+// it is one extra GET per app on this box.
+// - SEED-DON'T-REPLAY, per app, on this object's own mark. A box that has
+// never looked records where the log stands and cycles nothing.
+// - HIBERNATING routes are never woken. The root half skips them outright;
+// the agent's bounce is a no-op on a sleeping instance and cycles a warm
+// one. That is restart.go's rule verbatim, and narrowing the route set
+// cannot weaken it.
+// - A selector this binary does not understand addresses NOTHING. The
+// request is stepped over and the mark advances past it, so a future
+// `--region` cannot make today's box cycle an app.
+//
+// The one thing that is genuinely different: the CONTENT is read, not just the
+// etag. See internal/release/restartscope.go for why that grants nothing —
+// the account, the app and the candidate routes all come from this box's own
+// registry, and a selector can only ever pick a subset of them.
+package host
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "sort"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// restartMemory is the agent's in-process memory for BOTH restart signals: the
+// app-wide generation's etag, and the id of the last narrowed request acted
+// on. One struct rather than two loose maps because they are seeded together,
+// compared together on the same serial move, and are only ever meaningful as a
+// pair — a leg holding one and not the other would replay or drop half a
+// request set.
+//
+// It lives for the life of the agent process (fastConvergeLoop owns it,
+// restartSeed fills it), which is the whole point: there is nowhere else to
+// remember what "changed" means between two serial moves that may be days
+// apart.
+type restartMemory struct {
+ gen map[[2]string]string // app -> restart-gen.json etag, or genAbsent
+ scope map[[2]string]string // app -> last applied request id, or release.RestartScopeNone
+}
+
+func newRestartMemory() *restartMemory {
+ return &restartMemory{gen: map[[2]string]string{}, scope: map[[2]string]string{}}
+}
+
+// scopedRestart is one app's outstanding narrowed work: the requests this box
+// has not acted on yet, newest last.
+type scopedRestart struct {
+ app [2]string
+ requests []release.RestartRequest
+}
+
+// restartScopePass is the agent's comparison for narrowed requests — the twin
+// of restartGenPass, with the same first-sight, absent-forever and
+// read-error rules and for the same reasons. What it reports is not "this app
+// moved" but WHICH REQUESTS are outstanding, because that is the only thing
+// that says which routes to touch.
+//
+// Two requests inside one poll window are two entries in the log, so the pair
+// is not collapsed into whichever landed last — the failure a bare generation
+// object would have here, and the reason the object is a log at all.
+//
+// A mark that is no longer IN the log (this box was away long enough for its
+// position to roll off) re-baselines and asks for nothing, with a line saying
+// so. Restart requests are point-in-time events: a dropped one is recovered by
+// typing the verb again, while a replayed one cycles production for something
+// somebody asked for last week.
+func restartScopePass(ctx context.Context, routes []registry.Route, store release.Store, cache map[[2]string]string) (out []scopedRestart, readErrs []error) {
+ apps := restartApps(routes)
+ live := make(map[[2]string]bool, len(apps))
+ for _, k := range apps {
+ live[k] = true
+ g, _, err := release.LoadRestartScopes(ctx, store, k[0], k[1])
+ if err != nil {
+ // Unreadable OR unparseable, one answer: leave the mark exactly as
+ // it was and evaluate the rest. Clearing it would turn a recovered
+ // blip into a bogus first sight; advancing it past a log this
+ // binary cannot read would drop the requests in it for good.
+ readErrs = append(readErrs, fmt.Errorf("%s/%s: %w", k[0], k[1], err))
+ continue
+ }
+ newest := g.Newest()
+ prev, seen := cache[k]
+ if !seen {
+ cache[k] = newest // first sight: record, ask for nothing
+ continue
+ }
+ if prev == newest {
+ continue
+ }
+ pending, found := g.PendingSince(prev)
+ cache[k] = newest
+ if !found {
+ log.Printf("agent: narrowed restarts for %s/%s: this box's position is no longer in the log — re-baselined, nothing cycled (run the verb again if something is still waiting)", k[0], k[1])
+ continue
+ }
+ if len(pending) > 0 {
+ out = append(out, scopedRestart{app: k, requests: pending})
+ }
+ }
+ for k := range cache {
+ if !live[k] {
+ delete(cache, k)
+ }
+ }
+ return out, readErrs
+}
+
+// scopedHosts resolves outstanding requests against this box's routes: which
+// hosts of that app the requests actually name, deduplicated.
+//
+// Deduplicated across the whole set, not per request, because two pending
+// requests that both name a host mean one cycle of it, not two — and a verb
+// whose entire job is stopping processes must not manufacture a second stop
+// out of an operator typing the same thing twice.
+//
+// An UNKNOWN selector kind is stepped over with a log line and addresses
+// nothing. That is the forward-compatibility rule this feature is built on:
+// a box that cannot understand a narrowing must never fall back to the wider
+// thing.
+//
+// keep is the caller's own route filter — the root pass and the agent reach
+// different processes and each passes its own — so this function decides
+// membership and nothing else.
+func scopedHosts(routes []registry.Route, app [2]string, requests []release.RestartRequest, keep func(registry.Route) bool) []string {
+ seen := map[string]bool{}
+ var out []string
+ for _, req := range requests {
+ // Asked of the KIND alone, once, before any route is considered: an
+ // app with no routes on this box must not look like an app whose
+ // selector we failed to understand.
+ if _, understood := req.Scope.Selects(registry.Route{}); !understood {
+ log.Printf("restart: narrowed request for %s/%s names selector %q, which this build does not understand — nothing cycled for it (upgrade carlos on this box)",
+ app[0], app[1], req.Scope.Kind)
+ continue
+ }
+ for _, rt := range routes {
+ if !release.Adoptable(rt) || rt.App != app[1] || release.RouteAccount(rt) != app[0] {
+ continue
+ }
+ if sel, _ := req.Scope.Selects(rt); !sel || seen[rt.Host] || !keep(rt) {
+ continue
+ }
+ seen[rt.Host] = true
+ out = append(out, rt.Host)
+ }
+ }
+ sort.Strings(out)
+ return out
+}
+
+// execBounceable is the complement of what rootRestartRoute's systemctl arms
+// can reach — no named unit, no unit backing — so the agent's half and the
+// root half never cycle the same process twice. Split out of bounceExecRoutes
+// so the narrowed half cannot drift from the app-wide one on which processes
+// belong to whom.
+func execBounceable(rt registry.Route) bool {
+ return rt.Unit == "" && rt.Backing != registry.BackingUnit
+}
+
+// bounceScopedRoutes cycles the agent's OWN children for the routes the
+// outstanding narrowed requests name. bounceExecRoutes' argument, one selector
+// narrower — including on hibernation: Bounce cannot start anything, so a
+// sleeping instance is untouched and a warm one gets the cycle that was asked
+// for.
+func bounceScopedRoutes(routes []registry.Route, work []scopedRestart, bounce func(host string) bool) int {
+ if bounce == nil || len(work) == 0 {
+ return 0
+ }
+ n := 0
+ for _, w := range work {
+ for _, host := range scopedHosts(routes, w.app, w.requests, execBounceable) {
+ if bounce(host) {
+ n++
+ log.Printf("agent: narrowed restart requested: bounced exec instance %s", host)
+ }
+ }
+ }
+ return n
+}
+
+// restartScopeLeg is the ROOT pass's narrowed half, called from inside
+// restartPass under the SAME lock and against the same marks file: for every
+// app whose request log has moved past this box's mark, cycle the routes those
+// requests name.
+//
+// already is the hosts the app-wide half has cycled on this pass. A route that
+// has just been restarted by a wider request must not be restarted a second
+// time by a narrower one that arrived in the same window — one pass, one cycle
+// per host.
+//
+// The rules are restartPass's, applied to this object:
+//
+// - An app this box has not recorded is recorded and NOT cycled.
+// - An unknown baseline records and dispatches nothing.
+// - A mark that rolled off the log re-baselines and dispatches nothing.
+// - HIBERNATING routes are skipped and not counted as failures: restarting a
+// process that is not running is meaningless, waking it to restart it
+// defeats hibernation for no gain, and it execs fresh on its next wake.
+// - A dispatch FAILURE still advances the mark. A request is a point-in-time
+// event, not a desired state; retrying it every hour forever is how one
+// wrong unit name becomes a permanent restart loop.
+// - The mark advances only AFTER the dispatch, so a pass that dies mid-cycle
+// retries rather than swallowing the request.
+func restartScopeLeg(ctx context.Context, routes []registry.Route, store release.Store, apps [][2]string,
+ marks *restartMarks, known bool, restart func(host string) error, already map[string]bool) (changed, readFailed bool) {
+ for _, k := range apps {
+ g, _, err := release.LoadRestartScopes(ctx, store, k[0], k[1])
+ if err != nil {
+ log.Printf("adopt: narrowed restarts %s/%s: %v (skipped this pass)", k[0], k[1], err)
+ readFailed = true
+ continue
+ }
+ newest := g.Newest()
+ key := restartMarkKey(k[0], k[1])
+ prev, seen := marks.Scopes[key]
+ if !known || !seen {
+ if !seen || prev != newest {
+ marks.Scopes[key] = newest
+ changed = true
+ }
+ continue
+ }
+ if prev == newest {
+ continue
+ }
+ pending, found := g.PendingSince(prev)
+ if !found {
+ log.Printf("adopt: narrowed restarts for %s/%s: this box's position is no longer in the log — re-baselined, nothing cycled", k[0], k[1])
+ marks.Scopes[key] = newest
+ changed = true
+ continue
+ }
+ dispatched := 0
+ for _, host := range scopedHosts(routes, k, pending, func(registry.Route) bool { return true }) {
+ if already[host] {
+ continue
+ }
+ rt, ok := routeByHost(routes, host)
+ if !ok {
+ continue
+ }
+ if rt.Hibernate {
+ log.Printf("adopt: narrowed restart for %s/%s: %s is hibernating — left asleep (it execs fresh on its next wake)", k[0], k[1], host)
+ continue
+ }
+ if err := restart(host); err != nil {
+ log.Printf("adopt: narrowed restart for %s/%s: %s did not cycle: %v", k[0], k[1], host, err)
+ continue
+ }
+ already[host] = true
+ dispatched++
+ }
+ // One line per APP, saying DISPATCHED rather than cycled, for the
+ // reason the app-wide half says it: an exec-backed route reaching
+ // rootRestartRoute lands on a try-restart of a carlos-app@ unit that
+ // does not exist — a deliberate no-op this pass cannot tell apart from
+ // a real one.
+ log.Printf("adopt: narrowed restart for %s/%s: %d request(s), dispatched to %d route(s)", k[0], k[1], len(pending), dispatched)
+ marks.Scopes[key] = newest
+ changed = true
+ }
+ return changed, readFailed
+}
+
+// routeByHost finds a route by host among the ones this box serves. Linear
+// because the route set is a box's own table, and a map built per call would
+// be the same walk with an allocation on top.
+func routeByHost(routes []registry.Route, host string) (registry.Route, bool) {
+ for _, rt := range routes {
+ if rt.Host == host {
+ return rt, true
+ }
+ }
+ return registry.Route{}, false
+}
diff --git a/internal/host/restartscope_test.go b/internal/host/restartscope_test.go
new file mode 100644
index 000000000..2e54ecd6d
--- /dev/null
+++ b/internal/host/restartscope_test.go
@@ -0,0 +1,577 @@
+package host
+
+// The box half of a narrowed restart (discussion #16): that a request naming
+// one route cycles one route, that the rules restart.go pays for — seed
+// don't replay, never wake a sleeper, one cycle per host per pass — survive
+// the narrowing, and that a selector this build cannot read cycles NOTHING.
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "slices"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// askScoped is a member's `carlos restart --host/--environment` landing in the
+// bucket.
+func askScoped(t *testing.T, ctx context.Context, s release.Store, acct, app, kind, value string) release.RestartRequest {
+ t.Helper()
+ req, err := release.AppendRestartScope(ctx, s, acct, app, release.RestartScope{Kind: kind, Value: value})
+ if err != nil {
+ t.Fatalf("AppendRestartScope %s/%s (%s %s): %v", acct, app, kind, value, err)
+ }
+ return req
+}
+
+// envRoute is a unit-backed instance bound to a named config bundle — the
+// shape `--environment` exists for, and the shape only the root pass can
+// cycle.
+func envRoute(host, acct, app, env string) registry.Route {
+ rt := unitRoute(host, acct, app)
+ rt.Environment = env
+ return rt
+}
+
+// titogoRoutes is the fixture the whole discussion is about, small: one app,
+// several routes, two bundles between them, one on nothing.
+func titogoRoutes() []registry.Route {
+ return []registry.Route{
+ envRoute("connect.example.com", "bab", "titogo", "relay"),
+ envRoute("relay2.example.com", "bab", "titogo", "relay"),
+ envRoute("platform.example.com", "bab", "titogo", "router"),
+ unitRoute("bill.example.com", "bab", "titogo"),
+ }
+}
+
+// ---- the agent's sweep -------------------------------------------------------
+
+// TestRestartScopePassFirstSightIsSilent: a box that has never looked records
+// where the log stands and asks for nothing. Without this, a box coming up
+// after a request would cycle for it a second time — and a box whose memory
+// was lost would cycle for every request still in the log.
+func TestRestartScopePassFirstSightIsSilent(t *testing.T) {
+ ctx, s := genKit(t)
+ askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "bill.example.com")
+ cache := map[[2]string]string{}
+ work, errs := restartScopePass(ctx, titogoRoutes(), s, cache)
+ if len(work) != 0 || len(errs) != 0 {
+ t.Fatalf("first sight asked for %v (errs %v) — it must only record", work, errs)
+ }
+ if len(cache) != 1 {
+ t.Fatalf("cache = %v, want the app recorded", cache)
+ }
+}
+
+// TestRestartScopePassCarriesEveryRequestInAWindow is why the object is a log:
+// two requests between two sweeps are TWO selectors, and collapsing them into
+// the last one would silently drop a restart somebody was told was requested.
+func TestRestartScopePassCarriesEveryRequestInAWindow(t *testing.T) {
+ ctx, s := genKit(t)
+ routes := titogoRoutes()
+ cache := map[[2]string]string{}
+ restartScopePass(ctx, routes, s, cache) // baseline
+
+ askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "bill.example.com")
+ askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "router")
+ work, _ := restartScopePass(ctx, routes, s, cache)
+ if len(work) != 1 || len(work[0].requests) != 2 {
+ t.Fatalf("work = %+v, want one app carrying both requests", work)
+ }
+ hosts := scopedHosts(routes, work[0].app, work[0].requests, func(registry.Route) bool { return true })
+ if !slices.Equal(hosts, []string{"bill.example.com", "platform.example.com"}) {
+ t.Fatalf("addressed %v, want exactly the two named routes", hosts)
+ }
+ // And an unmoved log asks for nothing on the next sweep.
+ if work, _ := restartScopePass(ctx, routes, s, cache); len(work) != 0 {
+ t.Fatalf("an unmoved log asked for %+v", work)
+ }
+}
+
+// TestRestartScopePassReBaselinesALostPosition is the agent's own half of the
+// rolled-off-mark rule, exercised directly rather than through the root pass:
+// a box away long enough for its position to leave the log has no honest
+// answer to "what did I miss", so it records where the log now stands and asks
+// for nothing. Replaying instead would cycle routes for requests served days
+// ago; a dropped one is recovered by typing the verb again.
+func TestRestartScopePassReBaselinesALostPosition(t *testing.T) {
+ ctx, s := genKit(t)
+ routes := titogoRoutes()
+ req := askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "relay")
+
+ // This box acted up to a request that has since been trimmed off the front
+ // of the log.
+ cache := map[[2]string]string{{"bab", "titogo"}: "rolled-off"}
+ work, errs := restartScopePass(ctx, routes, s, cache)
+ if len(errs) != 0 {
+ t.Fatalf("unexpected read errors: %v", errs)
+ }
+ if len(work) != 0 {
+ t.Fatalf("a lost position asked for %+v — it must re-baseline instead of replaying the log", work)
+ }
+ if got := cache[[2]string{"bab", "titogo"}]; got != req.ID {
+ t.Fatalf("cache = %q, want the re-baseline to the log's newest %q", got, req.ID)
+ }
+ // And the NEXT request is acted on: re-baselining is not a box giving up.
+ next := askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "bill.example.com")
+ work, _ = restartScopePass(ctx, routes, s, cache)
+ if len(work) != 1 || len(work[0].requests) != 1 || work[0].requests[0].ID != next.ID {
+ t.Fatalf("work = %+v, want the one request made after the re-baseline", work)
+ }
+}
+
+// TestScopedHostsNarrowsToTheSelector is the whole complaint, tested: one
+// credential delivered to one bundle must cycle the routes reading that
+// bundle, not every route the app owns.
+func TestScopedHostsNarrowsToTheSelector(t *testing.T) {
+ routes := titogoRoutes()
+ app := [2]string{"bab", "titogo"}
+ all := func(registry.Route) bool { return true }
+
+ got := scopedHosts(routes, app, []release.RestartRequest{
+ {ID: "1", Scope: release.RestartScope{Kind: release.RestartScopeEnvironment, Value: "relay"}},
+ }, all)
+ if !slices.Equal(got, []string{"connect.example.com", "relay2.example.com"}) {
+ t.Fatalf("environment relay addressed %v, want the two relay routes and nothing else", got)
+ }
+ got = scopedHosts(routes, app, []release.RestartRequest{
+ {ID: "1", Scope: release.RestartScope{Kind: release.RestartScopeHost, Value: "bill.example.com"}},
+ }, all)
+ if !slices.Equal(got, []string{"bill.example.com"}) {
+ t.Fatalf("host selector addressed %v, want the one route", got)
+ }
+ // Another account's route with the same host is not this app's route.
+ other := append(routes, envRoute("connect.example.com", "zzz", "titogo", "relay"))
+ got = scopedHosts(other, app, []release.RestartRequest{
+ {ID: "1", Scope: release.RestartScope{Kind: release.RestartScopeHost, Value: "connect.example.com"}},
+ }, all)
+ if !slices.Equal(got, []string{"connect.example.com"}) {
+ t.Fatalf("addressed %v — a host must be resolved within this account and app only", got)
+ }
+}
+
+// TestScopedHostsIgnoresAnUnknownSelector: the forward-compatibility rule at
+// the point it matters. A `--region` invented after this binary shipped must
+// address nothing here, never fall back to the app.
+func TestScopedHostsIgnoresAnUnknownSelector(t *testing.T) {
+ got := scopedHosts(titogoRoutes(), [2]string{"bab", "titogo"}, []release.RestartRequest{
+ {ID: "1", Scope: release.RestartScope{Kind: "region", Value: "eu-west-1"}},
+ }, func(registry.Route) bool { return true })
+ if len(got) != 0 {
+ t.Fatalf("an unreadable selector addressed %v — a box must never widen what it cannot understand", got)
+ }
+}
+
+// TestScopedHostsDeduplicates: two requests naming the same route mean one
+// cycle of it. A verb whose job is stopping processes must not manufacture a
+// second stop out of somebody typing the same thing twice.
+func TestScopedHostsDeduplicates(t *testing.T) {
+ got := scopedHosts(titogoRoutes(), [2]string{"bab", "titogo"}, []release.RestartRequest{
+ {ID: "1", Scope: release.RestartScope{Kind: release.RestartScopeHost, Value: "connect.example.com"}},
+ {ID: "2", Scope: release.RestartScope{Kind: release.RestartScopeEnvironment, Value: "relay"}},
+ }, func(registry.Route) bool { return true })
+ if !slices.Equal(got, []string{"connect.example.com", "relay2.example.com"}) {
+ t.Fatalf("addressed %v, want each route once", got)
+ }
+}
+
+// TestBounceScopedRoutesCyclesOnlyItsOwnChildren: the agent's half reaches
+// exec children and nothing else, exactly as the app-wide bounce does — the
+// two halves must never cycle one process twice.
+func TestBounceScopedRoutesCyclesOnlyItsOwnChildren(t *testing.T) {
+ routes := []registry.Route{
+ execRoute("exec.example.com", "bab", "titogo"),
+ unitRoute("unit.example.com", "bab", "titogo"),
+ }
+ routes[0].Environment, routes[1].Environment = "relay", "relay"
+ var bounced []string
+ n := bounceScopedRoutes(routes, []scopedRestart{{
+ app: [2]string{"bab", "titogo"},
+ requests: []release.RestartRequest{{ID: "1", Scope: release.RestartScope{Kind: release.RestartScopeEnvironment, Value: "relay"}}},
+ }}, func(h string) bool { bounced = append(bounced, h); return true })
+ if n != 1 || !slices.Equal(bounced, []string{"exec.example.com"}) {
+ t.Fatalf("bounced %v (n=%d), want the exec child only — the unit route is root's", bounced, n)
+ }
+}
+
+// TestRestartRequestLegCarriesBothSignals: an app-wide request and a narrowed
+// one can land in the same poll window, and the leg that stopped at the first
+// would leave the other to the hourly floor.
+func TestRestartRequestLegCarriesBothSignals(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ routes := []registry.Route{
+ execRoute("a.example.com", "bab", "eleven"),
+ execRoute("b.example.com", "bab", "eleven"),
+ }
+ routes[1].Environment = "relay"
+ mem := newRestartMemory()
+ var bounced []string
+ bounce := func(h string) bool { bounced = append(bounced, h); return true }
+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, mem); asked {
+ t.Fatal("the baselining leg asked for a root pass")
+ }
+ askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeEnvironment, "relay")
+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, mem); !asked {
+ t.Fatal("a narrowed request did not ask for a root pass")
+ }
+ if !slices.Equal(bounced, []string{"b.example.com"}) {
+ t.Fatalf("bounced %v, want only the route reading relay", bounced)
+ }
+ if _, err := os.Stat(filepath.Join(dd, restartRequestMarker)); err != nil {
+ t.Fatalf("no marker after a narrowed request: %v", err)
+ }
+}
+
+// ---- the root pass -----------------------------------------------------------
+
+// TestRestartPassCyclesOnlyTheNarrowedRoutes is the fix itself: a request
+// naming one environment cycles the routes on it, and leaves the other
+// seventeen production routes running.
+func TestRestartPassCyclesOnlyTheNarrowedRoutes(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ routes := titogoRoutes()
+ marks := restartMarksPath(dd)
+
+ restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn) // baseline
+ askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "relay")
+ rec := &recordingRestart{}
+ restartPass(ctx, routes, s, marks, rec.fn)
+ if !slices.Equal(rec.hosts, []string{"connect.example.com", "relay2.example.com"}) {
+ t.Fatalf("cycled %v, want only the routes reading relay", rec.hosts)
+ }
+ // And the request is not served twice.
+ rec2 := &recordingRestart{}
+ restartPass(ctx, routes, s, marks, rec2.fn)
+ if len(rec2.hosts) != 0 {
+ t.Fatalf("a second pass cycled %v for a request already served", rec2.hosts)
+ }
+}
+
+// TestRestartPassSeedsNarrowedRequestsWithoutReplaying: a box that has never
+// looked at the log records where it stands and cycles nothing — the property
+// that keeps a lost marks file from bouncing a box's whole route table.
+func TestRestartPassSeedsNarrowedRequestsWithoutReplaying(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ routes := titogoRoutes()
+ req := askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "bill.example.com")
+
+ rec := &recordingRestart{}
+ restartPass(ctx, routes, s, restartMarksPath(dd), rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("the seeding pass cycled %v", rec.hosts)
+ }
+ m := marksOf(t, restartMarksPath(dd))
+ if got := m.Scopes[restartMarkKey("bab", "titogo")]; got != req.ID {
+ t.Fatalf("scope mark = %q, want the newest request %q", got, req.ID)
+ }
+}
+
+// TestRestartPassLeavesAHibernatingRouteAsleep: the load-bearing one. A
+// narrowed request that names a sleeping instance must not wake it — the
+// instance execs fresh on its next wake, which IS the restart.
+func TestRestartPassLeavesAHibernatingRouteAsleep(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ sleeper := envRoute("sleeper.example.com", "bab", "titogo", "relay")
+ sleeper.Hibernate = true
+ routes := append(titogoRoutes(), sleeper)
+ marks := restartMarksPath(dd)
+
+ restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn)
+ askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "relay")
+ rec := &recordingRestart{}
+ restartPass(ctx, routes, s, marks, rec.fn)
+ if slices.Contains(rec.hosts, "sleeper.example.com") {
+ t.Fatalf("cycled %v — a hibernating instance was woken by a restart request", rec.hosts)
+ }
+ if !slices.Equal(rec.hosts, []string{"connect.example.com", "relay2.example.com"}) {
+ t.Fatalf("cycled %v, want the two awake relay routes", rec.hosts)
+ }
+}
+
+// TestRestartPassCyclesEachHostOnceAcrossBothSignals: an app-wide request and
+// a narrowed one landing in the same window are one cycle per route, not two.
+func TestRestartPassCyclesEachHostOnceAcrossBothSignals(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ routes := titogoRoutes()
+ marks := restartMarksPath(dd)
+
+ restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn)
+ askRestart(t, ctx, s, "bab", "titogo")
+ askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "relay")
+ rec := &recordingRestart{}
+ restartPass(ctx, routes, s, marks, rec.fn)
+ slices.Sort(rec.hosts)
+ want := []string{"bill.example.com", "connect.example.com", "platform.example.com", "relay2.example.com"}
+ if !slices.Equal(rec.hosts, want) {
+ t.Fatalf("cycled %v, want each route exactly once: %v", rec.hosts, want)
+ }
+}
+
+// TestRestartPassIgnoresAnUnreadableSelector: the box-level half of the
+// never-widen rule. A request this build cannot interpret cycles nothing, and
+// the mark still advances so the pass does not re-read it hourly forever.
+func TestRestartPassIgnoresAnUnreadableSelector(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ routes := titogoRoutes()
+ marks := restartMarksPath(dd)
+ restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn)
+
+ // Written straight to the bucket, because AppendRestartScope refuses to
+ // write a kind it does not know — this is a NEWER console's request
+ // reaching an older box, which is the case that matters.
+ raw, err := json.Marshal(release.ScopedRestartGeneration{Schema: 1, Requests: []release.RestartRequest{
+ {ID: "future-1", Scope: release.RestartScope{Kind: "region", Value: "eu-west-1"}},
+ }})
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ if _, err := s.Put(ctx, release.RestartScopeKey("bab", "titogo"), raw, ""); err != nil {
+ t.Fatalf("put a future request: %v", err)
+ }
+ rec := &recordingRestart{}
+ restartPass(ctx, routes, s, marks, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("cycled %v for a selector this build cannot read — it must cycle nothing", rec.hosts)
+ }
+ if got := marksOf(t, marks).Scopes[restartMarkKey("bab", "titogo")]; got != "future-1" {
+ t.Fatalf("scope mark = %q, want the request stepped over rather than re-read forever", got)
+ }
+}
+
+// TestRestartPassReBaselinesALostPosition: a box away long enough for its mark
+// to roll off the log re-baselines and cycles nothing. A dropped request is
+// recoverable by typing the verb again; a replayed one is not.
+func TestRestartPassReBaselinesALostPosition(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ routes := titogoRoutes()
+ marks := restartMarksPath(dd)
+ restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn)
+
+ raw, err := json.Marshal(release.ScopedRestartGeneration{Schema: 1, Requests: []release.RestartRequest{
+ {ID: "gone", Scope: release.RestartScope{Kind: release.RestartScopeHost, Value: "bill.example.com"}},
+ }})
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ if _, err := s.Put(ctx, release.RestartScopeKey("bab", "titogo"), raw, ""); err != nil {
+ t.Fatalf("seed a log: %v", err)
+ }
+ // This box acted up to a request that has since been trimmed away.
+ m := marksOf(t, marks)
+ m.Scopes[restartMarkKey("bab", "titogo")] = "rolled-off"
+ saveMarks(t, marks, m)
+
+ rec := &recordingRestart{}
+ restartPass(ctx, routes, s, marks, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("cycled %v after losing its position — it must re-baseline instead", rec.hosts)
+ }
+ if got := marksOf(t, marks).Scopes[restartMarkKey("bab", "titogo")]; got != "gone" {
+ t.Fatalf("scope mark = %q, want the re-baseline to the log's newest", got)
+ }
+}
+
+// ---- the handover on a box that predates narrowing ---------------------------
+
+// writePreNarrowingMarks puts on disk exactly what the binary before this
+// feature wrote: schema, apps, and no `scopes` key at all. Every box that has
+// ever run the app-wide restart has one of these.
+func writePreNarrowingMarks(t *testing.T, path string, apps map[string]string) {
+ t.Helper()
+ raw, err := json.Marshal(map[string]any{"schema": 1, "apps": apps})
+ if err != nil {
+ t.Fatalf("marshal old marks: %v", err)
+ }
+ if err := os.WriteFile(path, raw, 0o644); err != nil {
+ t.Fatalf("write old marks: %v", err)
+ }
+}
+
+// armAgent is one agent start: the two silent sweeps that fill its memory, and
+// the seed that either writes the root pass's baseline or reports that the one
+// already on disk is missing a narrowed mark.
+func armAgent(t *testing.T, ctx context.Context, routes []registry.Route, s release.Store, dd string) (*restartMemory, bool) {
+ t.Helper()
+ mem := newRestartMemory()
+ if _, errs := restartGenPass(ctx, routes, s, mem.gen); len(errs) != 0 {
+ t.Fatalf("gen sweep: %v", errs)
+ }
+ if _, errs := restartScopePass(ctx, routes, s, mem.scope); len(errs) != 0 {
+ t.Fatalf("scope sweep: %v", errs)
+ }
+ need, err := seedRestartMarks(restartMarksPath(dd), mem)
+ if err != nil {
+ t.Fatalf("seed: %v", err)
+ }
+ return mem, need
+}
+
+// TestAPreNarrowingMarksFileIsCompletedBeforeTheNextRequestIsDropped is the
+// shape of every production box on the day this rolls, and the drop it used to
+// cause.
+//
+// The marks file was written before narrowed restarts existed, so it has
+// `apps` and no `scopes`. The agent's create-only seed will not touch it,
+// which leaves the ROOT pass with no narrowed position — and a missing
+// position reads as a FIRST SIGHT, so that pass records the log's newest id
+// and cycles nothing. An operator who ran `carlos restart --host relay2` in
+// that window was told "restart requested" and got no restart: the exec-shaped
+// routes survive because the agent bounces its own children out of memory, and
+// the unit-shaped ones — the relay and the router these flags exist for — are
+// the ones that vanish.
+//
+// The fix is the seed REPORTING the gap so that one root pass records the
+// marks. This walks the whole handover: arm, let that pass run, then ask.
+func TestAPreNarrowingMarksFileIsCompletedBeforeTheNextRequestIsDropped(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ routes := titogoRoutes() // unit-shaped: only the root pass can cycle these
+ marks := restartMarksPath(dd)
+ // A request from before the roll, already in the log. Nothing here may
+ // replay it.
+ askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "router")
+ writePreNarrowingMarks(t, marks, map[string]string{
+ restartMarkKey("bab", "titogo"): genAbsent, // no app-wide object; the old binary recorded the absence
+ })
+
+ if _, need := armAgent(t, ctx, routes, s, dd); !need {
+ t.Fatal("the agent saw a marks file with no narrowed mark and asked for nothing — " +
+ `the next narrowed request would be answered "requested" and cycle nothing unit-shaped`)
+ }
+ if got := marksOf(t, marks).Scopes; len(got) != 0 {
+ t.Fatalf("the seed wrote %v into an existing marks file — it must only report, never race the root pass's own write", got)
+ }
+
+ // The root pass that touch asks for. It records and cycles nothing: the
+ // request already in the log is this box's history, not work it owes.
+ rec := &recordingRestart{}
+ restartPass(ctx, routes, s, marks, rec.fn)
+ if len(rec.hosts) != 0 {
+ t.Fatalf("the baselining pass cycled %v — a box completing its baseline must replay nothing", rec.hosts)
+ }
+ if marksOf(t, marks).Scopes[restartMarkKey("bab", "titogo")] == "" {
+ t.Fatal("the pass recorded no scope mark, so the box would re-seed forever")
+ }
+
+ // And NOW the operator types the verb. This is the request that used to be
+ // dropped.
+ askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "relay2.example.com")
+ rec = &recordingRestart{}
+ restartPass(ctx, routes, s, marks, rec.fn)
+ if !slices.Equal(rec.hosts, []string{"relay2.example.com"}) {
+ t.Fatalf("cycled %v, want the one route the operator named", rec.hosts)
+ }
+
+ // A second agent start over a now-complete baseline asks for nothing: the
+ // repair is one pass per box, not one per agent start.
+ if _, need := armAgent(t, ctx, routes, s, dd); need {
+ t.Fatal("an agent restarting over a complete baseline asked for another root pass")
+ }
+}
+
+// TestSeedOnAFreshBoxAsksForNoRootPass is the genuinely-first-sight case, kept
+// separate because it is the one the create-only seed already handles: with no
+// marks file at all the agent writes BOTH memories itself, so there is nothing
+// left for a root pass to fill in — and the next narrowed request is acted on.
+func TestSeedOnAFreshBoxAsksForNoRootPass(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ routes := titogoRoutes()
+ marks := restartMarksPath(dd)
+ askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "router") // history
+
+ if _, need := armAgent(t, ctx, routes, s, dd); need {
+ t.Fatal("a seed that wrote the whole baseline itself still asked for a root pass")
+ }
+ if marksOf(t, marks).Scopes[restartMarkKey("bab", "titogo")] == "" {
+ t.Fatal("the fresh seed recorded no scope mark")
+ }
+
+ askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "bill.example.com")
+ rec := &recordingRestart{}
+ restartPass(ctx, routes, s, marks, rec.fn)
+ if !slices.Equal(rec.hosts, []string{"bill.example.com"}) {
+ t.Fatalf("cycled %v, want the named route — the seeded baseline is what the root pass trusts", rec.hosts)
+ }
+}
+
+// TestRestartSeedAsksTheRootPassThroughTheMarker is the same repair through
+// the real entry point, so the marker file — the only way this unprivileged
+// process can reach a root pass at all — is pinned rather than assumed.
+func TestRestartSeedAsksTheRootPassThroughTheMarker(t *testing.T) {
+ ctx, s := genKit(t)
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reg.Close()
+ for _, rt := range titogoRoutes() {
+ if err := reg.Upsert(rt); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ dd := t.TempDir()
+ writePreNarrowingMarks(t, restartMarksPath(dd), map[string]string{
+ restartMarkKey("bab", "titogo"): genAbsent,
+ })
+ restartSeed(ctx, reg, s, dd, newRestartMemory())
+ if _, err := os.Stat(filepath.Join(dd, restartRequestMarker)); err != nil {
+ t.Fatalf("no marker after arming over a pre-narrowing baseline: %v — "+
+ "the root pass would record no narrowed mark until the hourly floor", err)
+ }
+
+ // A baseline this agent wrote itself leaves the marker alone: an agent that
+ // touched it on every start would start a root pass on every start.
+ fresh := t.TempDir()
+ restartSeed(ctx, reg, s, fresh, newRestartMemory())
+ if _, err := os.Stat(filepath.Join(fresh, restartRequestMarker)); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("the fresh seed touched the marker (err %v), want no request at all", err)
+ }
+}
+
+// TestRestartPassAdvancesTheScopeMarkOnAFailedDispatch: a request is a
+// point-in-time event, not a desired state. A unit that will not come back
+// must not be retried on every pass forever for something somebody asked once.
+func TestRestartPassAdvancesTheScopeMarkOnAFailedDispatch(t *testing.T) {
+ ctx, s := genKit(t)
+ dd := t.TempDir()
+ routes := titogoRoutes()
+ marks := restartMarksPath(dd)
+ restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn)
+
+ req := askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "bill.example.com")
+ rec := &recordingRestart{fail: map[string]error{"bill.example.com": os.ErrPermission}}
+ restartPass(ctx, routes, s, marks, rec.fn)
+ if !slices.Equal(rec.hosts, []string{"bill.example.com"}) {
+ t.Fatalf("tried %v, want the named route", rec.hosts)
+ }
+ if got := marksOf(t, marks).Scopes[restartMarkKey("bab", "titogo")]; got != req.ID {
+ t.Fatalf("scope mark = %q after a failed dispatch, want %q — an unadvanced mark retries forever", got, req.ID)
+ }
+}
+
+// saveMarks rewrites the marks file the way the pass does, so a test can put
+// this box's memory where it needs it.
+func saveMarks(t *testing.T, path string, m *restartMarks) {
+ t.Helper()
+ raw, err := json.Marshal(m)
+ if err != nil {
+ t.Fatalf("marshal marks: %v", err)
+ }
+ if err := os.WriteFile(path, raw, 0o644); err != nil {
+ t.Fatalf("write marks: %v", err)
+ }
+}
diff --git a/internal/release/restartscope.go b/internal/release/restartscope.go
new file mode 100644
index 000000000..741fb508d
--- /dev/null
+++ b/internal/release/restartscope.go
@@ -0,0 +1,314 @@
+// The NARROWED restart signal (discussion #16): a request to cycle ONE route,
+// or the routes reading ONE config environment, rather than every instance an
+// app owns.
+//
+// Why a second object rather than a field on restart-gen.json. A box acts on
+// that object's ETAG MOVING — it never reads the content — so a selector
+// written INTO it would be invisible to every box running today's binary, and
+// those boxes would read the movement the only way they know how: restart the
+// whole app. For titogo that is nineteen production routes cycled to deliver
+// one credential to one of them. The rule this repo already has for wire
+// shapes ("anything written before your change must still parse") has a twin
+// that this file is built on: an old reader that cannot understand a NARROWED
+// request must do NOTHING, never something wider than was asked for.
+//
+// A sibling key gets that for free. Nothing that exists today reads
+// restart-scope.json, so:
+//
+// - OLD box, NEW (narrowed) request — never looks at the key, sees no
+// movement on restart-gen.json, cycles nothing. The request simply does
+// not reach it. The console says which routes it addressed and on which
+// box's registry it resolved them, so an operator whose fleet still has
+// an old box can see that it is one box's answer.
+// - NEW box, OLD (app-wide) request — restart-gen.json moves exactly as it
+// always did and this file takes no part.
+// - NEW box, FUTURE request kind — an entry whose `kind` this binary does
+// not know is SKIPPED, not widened (see RestartScope.Selects). A selector
+// nobody here understands addresses nothing.
+//
+// Bucket layout addition:
+//
+// apps/<acct>/<app>/restart-scope.json CAS-guarded, schema-1 request log
+//
+// It is a LOG rather than a single generation, which is the one place this
+// shape costs more than restart-gen.json's. Movement is enough to carry "some
+// restart was asked for"; it is not enough to carry WHICH routes, because two
+// requests inside one 2s poll window collapse into one etag move and the
+// earlier selector would be lost — silently, having told its caller
+// "requested". Recording each request with an id, and remembering the last id
+// acted on, makes a coalesced pair two requests again.
+//
+// The content is TRUSTED here in a way restart-gen.json's is not, so it is
+// worth saying exactly how far that goes: an entry can only ever NARROW. The
+// box matches entries against routes it already serves for that account and
+// app — the account, the app and the route set come from the box's own
+// registry, never from this object — so the worst a forged or corrupted entry
+// can do is name a selector matching every route of one app, which is
+// precisely what the unsigned app-wide object next to it already permits.
+// Nothing about what runs, or about which account owns what, is decided here.
+package release
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// The selector kinds. Only Host and Environment are ever written into the
+// object; App names an app-wide request, which is restart-gen.json's job and
+// travels no further than the HTTP door and the CLI that talks to it. It
+// lives here so the three of them spell it the same way.
+const (
+ RestartScopeHost = "host"
+ RestartScopeEnvironment = "environment"
+ RestartScopeApp = "app"
+)
+
+// restartScopeKeep bounds the log. Entries older than the last few dozen are
+// dropped on the next append: a box that has been away long enough to miss
+// them re-baselines (see PendingSince) rather than replaying a week of
+// requests, and the object stays a small, cheap read.
+const restartScopeKeep = 32
+
+// RestartScopeNone is the mark a box records for "I have looked and there is
+// nothing here" — the object is absent, or present with an empty log. The two
+// collapse deliberately: an object that appears carrying its first request
+// moves the mark from this sentinel to that request's id, which is movement,
+// which is exactly what it is.
+//
+// The NUL byte is what makes it safe as a sentinel — no request id contains
+// one, since ids are hex — and it is the same trick the box's own absent
+// marker uses for etags.
+const RestartScopeNone = "\x00none"
+
+// RestartScope is one selector: what to narrow a restart to.
+//
+// Kind is compared, never interpreted — an unknown one addresses nothing.
+// Value is matched against a registry column and never becomes a bucket key,
+// a filesystem path or a shell word; the strict shape checks belong at the
+// door that accepts it from a person, where a typo can still be answered with
+// what actually exists.
+type RestartScope struct {
+ Kind string `json:"kind"`
+ Value string `json:"value"`
+}
+
+// RestartRequest is one recorded request. ID is what a box remembers; nothing
+// is derived from it beyond "have I acted on this one".
+type RestartRequest struct {
+ ID string `json:"id"`
+ RequestedAt string `json:"requested_at"`
+ Scope RestartScope `json:"scope"`
+}
+
+// ScopedRestartGeneration is the object: apps/<acct>/<app>/restart-scope.json.
+// Requests are OLDEST FIRST, so the newest is the last element and a box's
+// "what have I not acted on" is a suffix.
+type ScopedRestartGeneration struct {
+ Schema int `json:"schema"` // 1
+ Requests []RestartRequest `json:"requests"`
+}
+
+// RestartScopeKey is the bucket key of app's narrowed-request log, derived
+// from AppPrefix like every other per-app key.
+func RestartScopeKey(acct, app string) string { return AppPrefix(acct, app) + "restart-scope.json" }
+
+// Selects reports whether rt is addressed by this scope, and whether this
+// binary understands the scope's kind at all.
+//
+// The second return is the forward-compatibility hinge, and the reason this
+// is two bools rather than one. A selector kind added after this binary was
+// built must address NOTHING here — not everything — so a caller that gets
+// understood=false skips the whole request rather than falling back to the
+// app. Widening on ignorance is the failure this whole file exists to avoid.
+//
+// The caller has already narrowed to the right account and app: this decides
+// only which of THOSE routes the selector names.
+func (sc RestartScope) Selects(rt registry.Route) (selected, understood bool) {
+ switch sc.Kind {
+ case RestartScopeHost:
+ return rt.Host == sc.Value, true
+ case RestartScopeEnvironment:
+ // An environment selector never matches the unbound routes: "" is not
+ // an environment name, it is the absence of one, and `--environment ""`
+ // cannot be typed. A request to cycle what reads the DEFAULT bundle is
+ // an app-wide request, and has the app-wide object.
+ return sc.Value != "" && rt.Environment == sc.Value, true
+ }
+ return false, false
+}
+
+// ValidRestartScope refuses a selector this package would write but no box
+// could act on. It is the last gate before the bucket, not the first: the
+// door that took the value from a person has already checked its shape and
+// answered a typo with what exists.
+func ValidRestartScope(sc RestartScope) error {
+ switch sc.Kind {
+ case RestartScopeHost, RestartScopeEnvironment:
+ default:
+ return fmt.Errorf("restart scope: unknown kind %q", sc.Kind)
+ }
+ if sc.Value == "" {
+ return fmt.Errorf("restart scope: %s has no value", sc.Kind)
+ }
+ if len(sc.Value) > 253 || strings.ContainsAny(sc.Value, "\x00\n\r") {
+ return fmt.Errorf("restart scope: %s value is not usable", sc.Kind)
+ }
+ return nil
+}
+
+// LoadRestartScopes reads the log. found=false means the object is absent,
+// which is the ordinary state of almost every app and is legal forever.
+//
+// A newer schema is an ERROR rather than a best-effort parse: a box that
+// cannot be sure what a request means must do nothing, and its caller records
+// nothing, so the request is picked up whenever the box is upgraded rather
+// than dropped or guessed at.
+func LoadRestartScopes(ctx context.Context, s Store, acct, app string) (*ScopedRestartGeneration, bool, error) {
+ key := RestartScopeKey(acct, app)
+ raw, _, err := s.Get(ctx, key)
+ if errors.Is(err, os.ErrNotExist) {
+ return &ScopedRestartGeneration{Schema: 1}, false, nil
+ }
+ if err != nil {
+ return nil, false, err
+ }
+ g, err := parseRestartScopes(key, raw)
+ if err != nil {
+ return nil, false, err
+ }
+ return g, true, nil
+}
+
+func parseRestartScopes(key string, raw []byte) (*ScopedRestartGeneration, error) {
+ var g ScopedRestartGeneration
+ if err := json.Unmarshal(raw, &g); err != nil {
+ return nil, fmt.Errorf("restart scopes at %s: %w", key, err)
+ }
+ if g.Schema > 1 {
+ return nil, fmt.Errorf("restart scopes at %s: schema %d is newer than this binary understands — upgrade carlos", key, g.Schema)
+ }
+ return &g, nil
+}
+
+// Newest is the id of the last request in the log, or RestartScopeNone for an
+// empty one — the value a box records when it has nothing to act on.
+func (g *ScopedRestartGeneration) Newest() string {
+ if g == nil || len(g.Requests) == 0 {
+ return RestartScopeNone
+ }
+ return g.Requests[len(g.Requests)-1].ID
+}
+
+// PendingSince returns the requests recorded after mark — what a box that has
+// already acted up to mark still owes — and whether mark could be located at
+// all.
+//
+// found=false means the mark is an id that is no longer in the log: the box
+// has been away long enough for its position to roll off the end. There is no
+// honest answer to "what did I miss" then, and the two candidate guesses are
+// not equal — replaying the whole log would cycle routes for requests served
+// days ago, while re-baselining drops requests that are recoverable by typing
+// the verb again. Callers re-baseline; see the box's restart pass, which makes
+// the same choice everywhere else it is offered.
+//
+// RestartScopeNone as the mark is NOT an unknown position: it is a box that
+// looked and found nothing, so everything now in the log is new to it.
+func (g *ScopedRestartGeneration) PendingSince(mark string) (pending []RestartRequest, found bool) {
+ if g == nil {
+ return nil, true
+ }
+ if mark == RestartScopeNone {
+ return g.Requests, true
+ }
+ for i, r := range g.Requests {
+ if r.ID == mark {
+ return g.Requests[i+1:], true
+ }
+ }
+ if len(g.Requests) == 0 {
+ // The log emptied under a mark that named a request — only possible if
+ // the object was replaced wholesale. Unknown position, same answer.
+ return nil, false
+ }
+ return nil, false
+}
+
+// AppendRestartScope records one narrowed request and returns it.
+//
+// A CAS conflict is RETRIED here, which is the opposite of TouchRestartGen's
+// treatment of the same error, and the difference is the whole point of this
+// object. There, any touch means "restart this app", so a racing writer has
+// already fired the exact signal this call exists to fire and a retry would
+// only manufacture a second restart. Here the other writer's request names a
+// DIFFERENT set of routes: dropping ours would answer "requested" for a
+// restart that will never happen. So we re-read and append again.
+//
+// The retry budget is small on purpose. This is a human-paced verb; a
+// contended object means something is wrong that another lap will not fix,
+// and the caller gets an error rather than a lie.
+func AppendRestartScope(ctx context.Context, s Store, acct, app string, scope RestartScope) (RestartRequest, error) {
+ if !ValidAccount(acct) {
+ return RestartRequest{}, fmt.Errorf("restart: invalid account %q", acct)
+ }
+ if !ValidApp(app) {
+ return RestartRequest{}, fmt.Errorf("restart: invalid app %q", app)
+ }
+ if err := ValidRestartScope(scope); err != nil {
+ return RestartRequest{}, err
+ }
+ key := RestartScopeKey(acct, app)
+ id := make([]byte, 16)
+ if _, err := io.ReadFull(rand.Reader, id); err != nil {
+ return RestartRequest{}, err
+ }
+ req := RestartRequest{
+ ID: hex.EncodeToString(id),
+ RequestedAt: time.Now().UTC().Format(time.RFC3339),
+ Scope: scope,
+ }
+ var lastErr error
+ for attempt := 0; attempt < 3; attempt++ {
+ raw, etag, err := s.Get(ctx, key)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return RestartRequest{}, err
+ }
+ g := &ScopedRestartGeneration{Schema: 1}
+ if err == nil {
+ if g, err = parseRestartScopes(key, raw); err != nil {
+ // Refusing beats overwriting: a log this binary cannot read is
+ // one whose entries it would silently drop on the next write.
+ return RestartRequest{}, err
+ }
+ } else {
+ etag = "" // create-only
+ }
+ g.Schema = 1
+ g.Requests = append(g.Requests, req)
+ if len(g.Requests) > restartScopeKeep {
+ g.Requests = g.Requests[len(g.Requests)-restartScopeKeep:]
+ }
+ body, err := json.Marshal(g)
+ if err != nil {
+ return RestartRequest{}, err
+ }
+ if _, err := s.Put(ctx, key, body, etag); err != nil {
+ if errors.Is(err, ErrCASConflict) {
+ lastErr = err
+ continue
+ }
+ return RestartRequest{}, err
+ }
+ return req, nil
+ }
+ return RestartRequest{}, fmt.Errorf("restart: %s is being written by someone else — nothing was requested; try again: %w", key, lastErr)
+}
diff --git a/internal/release/restartscope_test.go b/internal/release/restartscope_test.go
new file mode 100644
index 000000000..caf3d9a8b
--- /dev/null
+++ b/internal/release/restartscope_test.go
@@ -0,0 +1,301 @@
+package release
+
+// The narrowed restart's WIRE SHAPE (discussion #16): what lands in the
+// bucket, what a reader makes of it, and — the properties the whole design
+// rests on — that an old reader is untouched by it and a new reader never
+// widens a request it cannot understand.
+
+import (
+ "context"
+ "encoding/json"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+func loadScopes(t *testing.T, s Store, acct, app string) *ScopedRestartGeneration {
+ t.Helper()
+ g, _, err := LoadRestartScopes(context.Background(), s, acct, app)
+ if err != nil {
+ t.Fatalf("LoadRestartScopes: %v", err)
+ }
+ return g
+}
+
+// TestAppendRestartScopeRecordsEachRequest: the object is a LOG, not a
+// generation. Two requests inside one poll window must both survive — the etag
+// would have collapsed them into whichever landed last, and the earlier
+// selector would vanish having been answered "requested".
+func TestAppendRestartScopeRecordsEachRequest(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ first, err := AppendRestartScope(ctx, s, "bab", "titogo", RestartScope{Kind: RestartScopeHost, Value: "bill.example.com"})
+ if err != nil {
+ t.Fatalf("first append: %v", err)
+ }
+ second, err := AppendRestartScope(ctx, s, "bab", "titogo", RestartScope{Kind: RestartScopeEnvironment, Value: "relay"})
+ if err != nil {
+ t.Fatalf("second append: %v", err)
+ }
+ if first.ID == second.ID {
+ t.Fatalf("both requests share id %q — a box could not tell them apart", first.ID)
+ }
+ if _, err := time.Parse(time.RFC3339, first.RequestedAt); err != nil {
+ t.Fatalf("RequestedAt %q is not RFC3339: %v", first.RequestedAt, err)
+ }
+ g := loadScopes(t, s, "bab", "titogo")
+ if g.Schema != 1 {
+ t.Fatalf("Schema = %d, want 1", g.Schema)
+ }
+ if len(g.Requests) != 2 || g.Requests[0].ID != first.ID || g.Requests[1].ID != second.ID {
+ t.Fatalf("log = %+v, want both requests, oldest first", g.Requests)
+ }
+ if g.Newest() != second.ID {
+ t.Fatalf("Newest = %q, want the last request %q", g.Newest(), second.ID)
+ }
+}
+
+// TestAppendRestartScopeLeavesTheAppWideObjectAlone is the ADDITIVE property
+// stated as a test: an old box watches restart-gen.json's etag and nothing
+// else, so a narrowed request must not touch it. If it did, every box in the
+// fleet that predates narrowing would read the movement the only way it knows
+// how — restart the whole app — which is the accident this feature prevents.
+func TestAppendRestartScopeLeavesTheAppWideObjectAlone(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ if err := TouchRestartGen(ctx, s, "bab", "titogo"); err != nil {
+ t.Fatalf("TouchRestartGen: %v", err)
+ }
+ before := restartEtag(t, s, "bab", "titogo")
+ if _, err := AppendRestartScope(ctx, s, "bab", "titogo", RestartScope{Kind: RestartScopeHost, Value: "bill.example.com"}); err != nil {
+ t.Fatalf("append: %v", err)
+ }
+ if after := restartEtag(t, s, "bab", "titogo"); after != before {
+ t.Fatalf("restart-gen etag moved from %q to %q on a narrowed request — every old box would cycle the whole app", before, after)
+ }
+}
+
+// TestRestartScopeAbsentIsLegal: an app nobody has narrowed a restart for has
+// no object, forever, and reading that is not an error.
+func TestRestartScopeAbsentIsLegal(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ g, found, err := LoadRestartScopes(context.Background(), s, "bab", "keymail")
+ if err != nil {
+ t.Fatalf("LoadRestartScopes on an absent object: %v", err)
+ }
+ if found {
+ t.Fatal("found = true for an app with no object")
+ }
+ if g.Newest() != RestartScopeNone {
+ t.Fatalf("Newest = %q, want the recorded-nothing sentinel", g.Newest())
+ }
+ pending, ok := g.PendingSince(RestartScopeNone)
+ if !ok || len(pending) != 0 {
+ t.Fatalf("PendingSince(none) = %v, %v — an absent object owes nobody a restart", pending, ok)
+ }
+}
+
+// TestPendingSinceReturnsOnlyWhatIsNew: a box that acted up to a mark owes
+// exactly the suffix after it, and nothing else.
+func TestPendingSinceReturnsOnlyWhatIsNew(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ var ids []string
+ for _, env := range []string{"relay", "router", "demo"} {
+ req, err := AppendRestartScope(ctx, s, "bab", "titogo", RestartScope{Kind: RestartScopeEnvironment, Value: env})
+ if err != nil {
+ t.Fatalf("append %s: %v", env, err)
+ }
+ ids = append(ids, req.ID)
+ }
+ g := loadScopes(t, s, "bab", "titogo")
+
+ // A box that has never looked records "none" and then owes everything.
+ pending, ok := g.PendingSince(RestartScopeNone)
+ if !ok || len(pending) != 3 {
+ t.Fatalf("PendingSince(none) = %d requests, %v; want all three", len(pending), ok)
+ }
+ pending, ok = g.PendingSince(ids[0])
+ if !ok || len(pending) != 2 || pending[0].Scope.Value != "router" {
+ t.Fatalf("PendingSince(first) = %+v, %v; want router and demo", pending, ok)
+ }
+ if pending, ok := g.PendingSince(ids[2]); !ok || len(pending) != 0 {
+ t.Fatalf("PendingSince(newest) = %+v, %v; want nothing owed", pending, ok)
+ }
+}
+
+// TestPendingSinceReportsALostPosition: a mark that has rolled off the end of
+// the log has no honest answer, and the caller must be told so rather than
+// handed a guess. Replaying the whole log would cycle production for requests
+// served days ago; the box re-baselines instead.
+func TestPendingSinceReportsALostPosition(t *testing.T) {
+ g := &ScopedRestartGeneration{Schema: 1, Requests: []RestartRequest{
+ {ID: "b", Scope: RestartScope{Kind: RestartScopeHost, Value: "b.example.com"}},
+ }}
+ if pending, ok := g.PendingSince("a-that-rolled-off"); ok || len(pending) != 0 {
+ t.Fatalf("PendingSince(lost) = %+v, %v; want nothing and a report that the position is unknown", pending, ok)
+ }
+}
+
+// TestRestartScopeLogIsBounded: the object stays small, and the oldest entries
+// are what go. A box far enough behind to lose its position re-baselines,
+// which is the safe direction.
+func TestRestartScopeLogIsBounded(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ var first RestartRequest
+ for i := 0; i < restartScopeKeep+5; i++ {
+ req, err := AppendRestartScope(ctx, s, "bab", "titogo", RestartScope{Kind: RestartScopeHost, Value: "a.example.com"})
+ if err != nil {
+ t.Fatalf("append %d: %v", i, err)
+ }
+ if i == 0 {
+ first = req
+ }
+ }
+ g := loadScopes(t, s, "bab", "titogo")
+ if len(g.Requests) != restartScopeKeep {
+ t.Fatalf("log holds %d requests, want it trimmed to %d", len(g.Requests), restartScopeKeep)
+ }
+ if _, ok := g.PendingSince(first.ID); ok {
+ t.Fatal("the oldest request survived the trim — the log is unbounded")
+ }
+}
+
+// TestRestartScopeSelectsNarrowsAndNeverWidens is the forward-compatibility
+// rule: an unknown selector kind addresses NOTHING and says it did not
+// understand, so a `--region` invented later cannot make this binary fall back
+// to cycling the app.
+func TestRestartScopeSelectsNarrowsAndNeverWidens(t *testing.T) {
+ relay := registry.Route{Host: "connect.example.com", App: "titogo", Environment: "relay"}
+ plain := registry.Route{Host: "bill.example.com", App: "titogo"}
+
+ if sel, ok := (RestartScope{Kind: RestartScopeHost, Value: "bill.example.com"}).Selects(plain); !sel || !ok {
+ t.Fatalf("host selector missed its own route (sel %v, understood %v)", sel, ok)
+ }
+ if sel, _ := (RestartScope{Kind: RestartScopeHost, Value: "bill.example.com"}).Selects(relay); sel {
+ t.Fatal("host selector matched another route")
+ }
+ if sel, ok := (RestartScope{Kind: RestartScopeEnvironment, Value: "relay"}).Selects(relay); !sel || !ok {
+ t.Fatalf("environment selector missed a bound route (sel %v, understood %v)", sel, ok)
+ }
+ // An unbound route reads the app's DEFAULT bundle. "" is the absence of an
+ // environment, not the name of one, so an environment selector never
+ // reaches it — otherwise a bundle-shaped request would silently become an
+ // app-wide one for every route nobody had bound.
+ if sel, _ := (RestartScope{Kind: RestartScopeEnvironment, Value: ""}).Selects(plain); sel {
+ t.Fatal("an empty environment selector matched an unbound route — that is an app-wide restart in disguise")
+ }
+ for _, rt := range []registry.Route{relay, plain} {
+ sel, understood := (RestartScope{Kind: "region", Value: "eu-west-1"}).Selects(rt)
+ if sel || understood {
+ t.Fatalf("a selector kind this build does not know addressed %s (sel %v, understood %v)", rt.Host, sel, understood)
+ }
+ }
+}
+
+// TestValidRestartScopeRefusesWhatNoBoxCouldAct: the last gate before the
+// bucket. A request nothing could ever match must not be written, or a box
+// would step over it forever.
+func TestValidRestartScopeRefusesWhatNoBoxCouldAct(t *testing.T) {
+ for _, sc := range []RestartScope{
+ {Kind: "", Value: "x"},
+ {Kind: "region", Value: "eu-west-1"},
+ {Kind: RestartScopeApp, Value: ""}, // app-wide has its own object
+ {Kind: RestartScopeHost, Value: ""},
+ {Kind: RestartScopeEnvironment, Value: "bad\nvalue"},
+ } {
+ if err := ValidRestartScope(sc); err == nil {
+ t.Fatalf("ValidRestartScope(%+v) accepted it", sc)
+ }
+ }
+ if err := ValidRestartScope(RestartScope{Kind: RestartScopeHost, Value: "a.example.com"}); err != nil {
+ t.Fatalf("a good host scope was refused: %v", err)
+ }
+}
+
+// TestRestartScopeRefusesANewerSchema: a log written by a console this binary
+// is older than must not be guessed at, and must not be OVERWRITTEN either —
+// appending to it would silently drop entries this build cannot see. Both
+// paths refuse, which leaves the request to be picked up after an upgrade.
+func TestRestartScopeRefusesANewerSchema(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ raw, err := json.Marshal(map[string]any{"schema": 2, "requests": []any{}})
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ if _, err := s.Put(ctx, RestartScopeKey("bab", "titogo"), raw, ""); err != nil {
+ t.Fatalf("seed a newer object: %v", err)
+ }
+ if _, _, err := LoadRestartScopes(ctx, s, "bab", "titogo"); err == nil {
+ t.Fatal("a schema-2 log parsed as though this build understood it")
+ }
+ if _, err := AppendRestartScope(ctx, s, "bab", "titogo", RestartScope{Kind: RestartScopeHost, Value: "a.example.com"}); err == nil {
+ t.Fatal("appended to a schema-2 log — entries this build cannot read would have been dropped")
+ }
+}
+
+// TestAppendRestartScopeRetriesACASConflict is the difference from
+// TouchRestartGen, which treats a conflict as success. There, any touch fires
+// the one signal that exists; here the other writer's request names a
+// DIFFERENT set of routes, so losing ours would answer "requested" for a
+// restart that never happens.
+func TestAppendRestartScopeRetriesACASConflict(t *testing.T) {
+ s := &conflictOnceStore{Store: NewDirStore(t.TempDir())}
+ ctx := context.Background()
+ if _, err := AppendRestartScope(ctx, s, "bab", "titogo", RestartScope{Kind: RestartScopeHost, Value: "a.example.com"}); err != nil {
+ t.Fatalf("append through one conflict: %v", err)
+ }
+ if s.conflicts != 1 {
+ t.Fatalf("the store served %d conflicts, want the one this test staged", s.conflicts)
+ }
+ if g := loadScopes(t, s.Store, "bab", "titogo"); len(g.Requests) != 1 {
+ t.Fatalf("log = %+v, want the retried request to have landed", g.Requests)
+ }
+}
+
+// conflictOnceStore fails the FIRST Put with a CAS conflict, as a racing
+// writer would, and behaves normally after that.
+type conflictOnceStore struct {
+ Store
+ conflicts int
+}
+
+func (c *conflictOnceStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
+ if c.conflicts == 0 {
+ c.conflicts++
+ return "", ErrCASConflict
+ }
+ return c.Store.Put(ctx, key, body, ifMatch)
+}
+
+// TestRestartScopeKeyIsUnderTheAppPrefix: everything an app owns lives under
+// one account-scoped prefix, permanently — trash, retention and per-app
+// deletion all depend on it.
+func TestRestartScopeKeyIsUnderTheAppPrefix(t *testing.T) {
+ if got, want := RestartScopeKey("bab", "titogo"), AppPrefix("bab", "titogo")+"restart-scope.json"; got != want {
+ t.Fatalf("RestartScopeKey = %q, want %q", got, want)
+ }
+ if RestartScopeKey("bab", "titogo") == RestartGenKey("bab", "titogo") {
+ t.Fatal("the narrowed log shares a key with the app-wide generation — an old box would read it as an app-wide request")
+ }
+}
+
+// TestAppendRestartScopeRefusesABadName keeps a hostile account or app out of
+// the key, the same gate TouchRestartGen applies.
+func TestAppendRestartScopeRefusesABadName(t *testing.T) {
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ if _, err := AppendRestartScope(ctx, s, "../etc", "titogo", RestartScope{Kind: RestartScopeHost, Value: "a.example.com"}); err == nil {
+ t.Fatal("a traversal-shaped account was accepted")
+ }
+ if _, err := AppendRestartScope(ctx, s, "bab", "../x", RestartScope{Kind: RestartScopeHost, Value: "a.example.com"}); err == nil {
+ t.Fatal("a traversal-shaped app was accepted")
+ }
+ if _, err := os.Stat(t.TempDir()); err != nil {
+ t.Fatalf("temp dir: %v", err)
+ }
+}
diff --git a/main.go b/main.go
index 4ec5ccee6..662b28bf5 100644
--- a/main.go
+++ b/main.go
@@ -167,7 +167,8 @@ Your apps — ship and run software on a CARLOS deployment
deploy ship + promote + wait until the URL serves the new version
canary the same, onto a host and channel of this branch's own, so
nothing the app already serves changes: deploy | ls | rm
- restart cycle an app's instances — no new version, no config change
+ restart cycle an app's instances — no new version, no config change;
+ --host or --environment narrows it, --dry-run names the set
schedule recurring or one-shot work per app: ls | set | rm | run
store object storage for an app: create | status | grant | rotate
email tenant email sending: enable | domains add | status | test |
diff --git a/restart.go b/restart.go
index 57bda6fbe..3edad0b3d 100644
--- a/restart.go
+++ b/restart.go
@@ -8,6 +8,31 @@
// Nothing is pushed at a box, nothing wakes a sleeping instance, and no
// release ladder or bake window is involved — there is no new version for one
// to gate.
+//
+// SELECTORS (discussion #16). The app was once the only thing this verb could
+// address, which for an app like titogo — one app, nineteen production routes,
+// relay and router and tunnel and every customer instance under it — made the
+// only available request "bounce all of production" when what had changed was
+// one credential in one bundle. So:
+//
+// --host <h> exactly that route.
+// --environment <e> every route reading that config bundle. The shape that
+// pairs with `carlos secrets set --environment relay`,
+// because a bundle is delivered to an ENVIRONMENT and
+// "restart what reads this" is the request that follows.
+// --app alone the whole app, byte for byte what it always did.
+//
+// PRECEDENCE: --host wins over --environment, but only after the console has
+// checked that the named host really does read the named environment —
+// silently ignoring the wider flag would hide the case where one of the two is
+// a typo. A selector that names nothing is REFUSED, with what the app actually
+// has; it is never widened. And --dry-run answers the question the reporter
+// could not: which routes would this stop, before any of them stop.
+//
+// The narrowed forms go to a DIFFERENT console door, which is a safety
+// property, not tidiness — see restartScoped in apimode.go. The old door reads
+// no body, so a console that predates this would answer a narrowed request by
+// cycling the whole app.
package main
import (
@@ -16,6 +41,8 @@ import (
"fmt"
"log"
"strings"
+
+ "github.com/carlosframework/platform/internal/release"
)
// runRestart is the verb. API mode ONLY, like deploy: the request is a console
@@ -23,12 +50,36 @@ import (
func runRestart(args []string) {
fs := flag.NewFlagSet("restart", flag.ExitOnError)
app := appFlag(fs, "app whose instances to cycle")
+ host := fs.String("host", "", "cycle exactly this route, instead of every instance of the app")
+ environment := environmentFlag(fs, "cycle every route reading this named config bundle, instead of every instance of the app")
+ dryRun := fs.Bool("dry-run", false, "name the routes this would cycle and stop; nothing is restarted")
acctRaw, acctIDRaw, _ := accountFlags(fs)
consoleFlag := fs.String("console", "", "console origin to restart through (API mode only; default: the only logged-in one)")
- fs.Usage = flagUsage(fs, "")
+ fs.Usage = flagUsage(fs, restartUsage)
fs.Parse(args)
name := strings.TrimSpace(*app)
- usage := "usage: carlos restart --app <a> [--account <sqid-or-name>] [--account-id <sqid>] [--console <origin>]"
+ hostSel := strings.TrimSpace(*host)
+ envSel := strings.TrimSpace(*environment)
+ usage := restartUsage
+ // An EMPTY selector that was actually typed is refused rather than treated
+ // as "no selector". `--host ""` (a shell variable that expanded to nothing
+ // is how it happens) would otherwise fall through to an app-wide restart —
+ // somebody asking to cycle one route and cycling every one instead, which
+ // is precisely the accident these flags exist to prevent, arriving through
+ // the flags themselves.
+ for _, f := range []struct{ name, value string }{
+ {"host", *host},
+ {"environment", *environment},
+ } {
+ if flagWasSet(fs, f.name) && strings.TrimSpace(f.value) == "" {
+ log.Fatalf("restart: --%s was given with an empty value — nothing was restarted. "+
+ "Name a route or a bundle, or drop the flag to cycle the whole app.", f.name)
+ }
+ }
+ // The same client-side gate `carlos env` applies to the same flag, for the
+ // same reason: a value the console is certain to refuse should cost a typo
+ // nothing.
+ checkEnvironmentFlag(envSel)
if name != "" {
// Client-side gate before the server's, the same one `carlos apps create`
// applies: a name the console is certain to refuse should cost a typo
@@ -42,6 +93,15 @@ func runRestart(args []string) {
printUsing(stdout, sess)
acct := resolvedAccount(sess, acctRaw, acctIDRaw)
name = resolvedApp(sess, acct, name, usage)
+ // Anything narrowed, and any dry run, goes to the door that can resolve a
+ // selector and name what it resolved to. A plain --app takes the original
+ // door, unchanged: it is the one form that works against every console in
+ // the fleet, including one that predates all of this, and an existing
+ // script must not start depending on a door its console may not serve.
+ if hostSel != "" || envSel != "" || *dryRun {
+ runRestartScoped(sess, acct, name, hostSel, envSel, *dryRun)
+ return
+ }
res, err := sess.restart(context.Background(), acct, name)
if err != nil {
log.Fatalf("restart: %v", err)
@@ -58,12 +118,98 @@ func runRestart(args []string) {
// and respawn on their next request or scheduled run — which for an idle
// app may be a while, and is the correct behaviour, not a delay to
// apologise for.
- fmt.Fprintf(stdout, "%s for %s/%s — running instances stop within seconds; "+
- "long-running ones restart, on-demand ones respawn on their next request "+
- "(sleeping instances stay asleep and start fresh when next woken)\n",
- colorFor(stdout).ok("restart requested"), res.Account, res.App)
+ fmt.Fprintf(stdout, "%s for %s/%s — %s\n",
+ colorFor(stdout).ok("restart requested"), res.Account, res.App, restartTimingCopy)
}
+// restartUsage is the one place the flags and their precedence are stated for
+// somebody at a terminal. Precedence is written down HERE and in
+// docs/site/cli.md rather than left to be inferred, because the failure this
+// verb was narrowed to prevent is somebody inferring wrongly.
+const restartUsage = `usage: carlos restart [--app <a>] [--host <h>] [--environment <e>] [--dry-run]
+ [--account <sqid|name>] [--account-id <sqid>] [--console <origin>]
+
+Cycles running processes with no new version and no config change.
+
+ --app alone every instance of the app.
+ --environment <e> only the routes reading that config bundle — what to
+ reach for after carlos secrets set --environment <e>.
+ --host <h> only that one route.
+
+--host wins over --environment when both are given, and the console refuses
+the pair if that host does not actually read that environment: one of the two
+is then a typo. A selector matching nothing is refused, never widened.
+
+--dry-run names the routes the same request would cycle and stops there.
+
+Signed-in only (carlos auth login).`
+
+// runRestartScoped is the narrowed and dry-run half: ask the console to
+// resolve the selector, then print WHAT IT RESOLVED TO. Naming the set is the
+// whole point of the verb having selectors at all — the complaint that got
+// them built was that an operator could not tell what the blast radius would
+// be, and a narrowed restart that reported no more than the old one would have
+// fixed the radius and left the blindness.
+func runRestartScoped(sess *apiSession, acct apiAcct, app, host, environment string, dryRun bool) {
+ res, err := sess.restartScoped(context.Background(), acct, app, host, environment, dryRun)
+ if err != nil {
+ log.Fatalf("restart: %v", err)
+ }
+ c := colorFor(stdout)
+ what := restartScopeWords(res.Scope.Kind, res.Scope.Value)
+ if res.Requested {
+ fmt.Fprintf(stdout, "%s for %s/%s — %s, %d route(s):\n",
+ c.ok("restart requested"), res.Account, res.App, what, len(res.Routes))
+ } else {
+ // Said FIRST, before the list, so a skimmed answer cannot be mistaken
+ // for a restart that happened.
+ fmt.Fprintf(stdout, "%s — nothing was restarted. This would cycle %d route(s) of %s/%s (%s):\n",
+ c.warn("dry run"), len(res.Routes), res.Account, res.App, what)
+ }
+ w := newTable(stdout)
+ fmt.Fprintln(w, "HOST\tENVIRONMENT\tUNIT\tNOTE")
+ for _, rt := range res.Routes {
+ note := "—"
+ if rt.Hibernate {
+ // Named rather than filtered out. A route in the set that will not
+ // cycle now is a fact the operator needs, and "asleep" is the
+ // correct outcome — it execs fresh when something next wakes it —
+ // not a failure to apologise for or to hide.
+ note = "asleep — left as it is, execs fresh on its next wake"
+ }
+ fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", rt.Host, dashIfEmpty(rt.Environment), dashIfEmpty(rt.Unit), note)
+ }
+ w.Flush()
+ if res.Requested {
+ fmt.Fprintln(stdout, restartTimingCopy)
+ }
+ // Said every time: this console reads its OWN box's registry, so on a
+ // multi-box fleet the list is what one box can see. The REQUEST is not
+ // limited that way — every box serving the app converges on it and applies
+ // the same selector to its own routes — and conflating the two would be
+ // its own lie.
+ mutedf(stdout, "(routes on this deployment's own box; other boxes in a fleet apply the same selector to their own routes and are not listed)")
+}
+
+// restartScopeWords renders a selector as the phrase both halves of the output
+// use, so the header and any refusal name the same thing the same way.
+func restartScopeWords(kind, value string) string {
+ switch kind {
+ case release.RestartScopeHost:
+ return "host " + value
+ case release.RestartScopeEnvironment:
+ return "environment " + value
+ }
+ return "the whole app"
+}
+
+// restartTimingCopy is the sentence the app-wide form has printed since #113,
+// kept identical here: what "requested" actually means for each shape of
+// instance. One string rather than two, because two copies of one sentence is
+// how they end up disagreeing.
+const restartTimingCopy = "running instances stop within seconds; long-running ones restart, " +
+ "on-demand ones respawn on their next request (sleeping instances stay asleep and start fresh when next woken)"
+
// requireRestartAPISession resolves signed-in mode for restart, refusing
// direct-bucket mode with restart's OWN message — the pattern
// requireDeployAPISession sets, and for its reason: a refusal that names
diff --git a/restart_test.go b/restart_test.go
index 1954c7a12..94a7dd19d 100644
--- a/restart_test.go
+++ b/restart_test.go
@@ -155,3 +155,185 @@ func TestRestartVerbRefusals(t *testing.T) {
t.Fatalf("bad app name = %q, want the name gate", out)
}
}
+
+// ---- the narrowed forms (discussion #16) ------------------------------------
+
+// TestAPISessionRestartScopedWireShape pins the SEPARATE path and the body it
+// carries. Separate is the safety property: the app-wide door reads no body at
+// all, so a narrowed request sent there would be answered by cycling every
+// instance the app owns. This client must never send one there.
+func TestAPISessionRestartScopedWireShape(t *testing.T) {
+ var gotPath, gotMethod string
+ var gotBody map[string]any
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotPath, gotMethod = r.URL.RequestURI(), r.Method
+ json.NewDecoder(r.Body).Decode(&gotBody)
+ json.NewEncoder(w).Encode(map[string]any{
+ "account": "bab", "app": "titogo", "requested": true, "dry_run": false,
+ "scope": map[string]string{"kind": "environment", "value": "relay"},
+ "routes": []map[string]any{
+ {"host": "connect.go.tito.io", "environment": "relay", "hibernate": false},
+ {"host": "sleepy.go.tito.io", "environment": "relay", "hibernate": true},
+ },
+ })
+ }))
+ defer ts.Close()
+
+ res, err := newTestSession(ts.URL).restartScoped(context.Background(),
+ apiAcct{Seg: "bab"}, "titogo", "", "relay", false)
+ if err != nil {
+ t.Fatalf("restartScoped: %v", err)
+ }
+ if gotMethod != http.MethodPost {
+ t.Fatalf("method = %s, want POST", gotMethod)
+ }
+ if gotPath != "/api/cli/apps/bab/titogo/restart/scope" {
+ t.Fatalf("path = %s — a narrowed request must not reach the app-wide door", gotPath)
+ }
+ if gotBody["environment"] != "relay" || gotBody["host"] != "" || gotBody["dry_run"] != false {
+ t.Fatalf("body = %v, want the selector and no dry run", gotBody)
+ }
+ // The ADDRESSED SET comes back, which is the whole point of the flags: the
+ // complaint was that an operator could not tell what the blast radius was.
+ if len(res.Routes) != 2 || res.Routes[0].Host != "connect.go.tito.io" || !res.Routes[1].Hibernate {
+ t.Fatalf("routes = %+v, want both rows and the sleeping one flagged", res.Routes)
+ }
+ if !res.Requested || res.Scope.Kind != "environment" || res.Scope.Value != "relay" {
+ t.Fatalf("result = %+v, want the console's own account of what it did", res)
+ }
+}
+
+// TestAPISessionRestartScopedDryRunAsksForNothing: the flag travels, and the
+// console's answer — not this client's own flags — is what says whether
+// anything was restarted.
+func TestAPISessionRestartScopedDryRunAsksForNothing(t *testing.T) {
+ var gotBody map[string]any
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ json.NewDecoder(r.Body).Decode(&gotBody)
+ json.NewEncoder(w).Encode(map[string]any{
+ "account": "bab", "app": "titogo", "requested": false, "dry_run": true,
+ "scope": map[string]string{"kind": "app"}, "routes": []map[string]any{},
+ })
+ }))
+ defer ts.Close()
+
+ res, err := newTestSession(ts.URL).restartScoped(context.Background(), apiAcct{Seg: "bab"}, "titogo", "", "", true)
+ if err != nil {
+ t.Fatalf("restartScoped dry run: %v", err)
+ }
+ if gotBody["dry_run"] != true {
+ t.Fatalf("body = %v, want the dry run asked for", gotBody)
+ }
+ if res.Requested {
+ t.Fatal("a dry run came back reporting a restart")
+ }
+}
+
+// TestAPISessionRestartScoped404NamesTheOldConsole: a console that predates
+// the narrowed door answers net/http's byte-identical "404 page not found",
+// exactly as the membership gate does. The message must name all three
+// possibilities — including the form that still works there, because telling
+// somebody to redeploy their console and nothing else leaves them stuck.
+func TestAPISessionRestartScoped404NamesTheOldConsole(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.NotFound(w, r)
+ }))
+ defer ts.Close()
+
+ _, err := newTestSession(ts.URL).restartScoped(context.Background(), apiAcct{Seg: "bab"}, "titogo", "bill.go.tito.io", "", false)
+ if err == nil {
+ t.Fatal("a 404 must error")
+ }
+ for _, want := range []string{"-account", "predates", "redeploy the console", "carlos restart --app"} {
+ if !strings.Contains(err.Error(), want) {
+ t.Fatalf("error %q does not mention %q", err, want)
+ }
+ }
+}
+
+// TestAPISessionRestartScopedServerRefusalSurfacesVerbatim: the refusal that
+// names what exists ("no route of this app reads environment relayy — this
+// console's box has relay, router") is the console's own sentence, and it is
+// the sentence that lets somebody spot their typo. It must not be replaced
+// with a generic one here.
+func TestAPISessionRestartScopedServerRefusalSurfacesVerbatim(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusBadRequest)
+ json.NewEncoder(w).Encode(map[string]string{
+ "error": "no route of this app reads environment relayy — this console's box has relay, router",
+ })
+ }))
+ defer ts.Close()
+
+ _, err := newTestSession(ts.URL).restartScoped(context.Background(), apiAcct{Seg: "bab"}, "titogo", "", "relayy", false)
+ if err == nil || !strings.Contains(err.Error(), "relay, router") {
+ t.Fatalf("restart scope 400 = %v, want the console's own list of what exists", err)
+ }
+}
+
+// TestRestartSelectorRefusals drives the real binary for the selector gates it
+// applies before dialling anything — the client-side half of "a typo should
+// cost nothing".
+func TestRestartSelectorRefusals(t *testing.T) {
+ bin := buildCarlos(t)
+ scratch := t.TempDir()
+ run := func(args ...string) (string, error) {
+ t.Helper()
+ cmd := exec.Command(bin, args...)
+ cmd.Env = carlosTestEnv(t)
+ cmd.Dir = scratch
+ out, err := cmd.CombinedOutput()
+ return string(out), err
+ }
+
+ // "default" is the base bundle no route is BOUND to, so a restart of what
+ // reads it is a restart of the app — refused by name rather than resolved
+ // to an empty set. Same gate `carlos env` applies to the same flag.
+ out, err := run("restart", "-app", "titogo", "-environment", "default")
+ if err == nil {
+ t.Fatalf("restart --environment default exited 0:\n%s", out)
+ }
+ if !strings.Contains(out, "already names the base bundle") {
+ t.Fatalf("--environment default = %q, want the named refusal", out)
+ }
+
+ out, err = run("restart", "-app", "titogo", "-environment", "Not Valid")
+ if err == nil {
+ t.Fatalf("restart with a malformed environment exited 0:\n%s", out)
+ }
+ if !strings.Contains(out, "not a valid environment name") {
+ t.Fatalf("bad environment = %q, want the shape gate", out)
+ }
+
+ // A selector that was TYPED but empty — a shell variable that expanded to
+ // nothing — must not fall through to "no selector", which is an app-wide
+ // restart. Somebody asking to cycle one route and cycling all of them
+ // instead is the accident the flags exist to prevent, arriving through the
+ // flags themselves.
+ for _, args := range [][]string{
+ {"restart", "-app", "titogo", "-host", ""},
+ {"restart", "-app", "titogo", "-environment", ""},
+ } {
+ out, err := run(args...)
+ if err == nil {
+ t.Fatalf("%v exited 0:\n%s", args, out)
+ }
+ if !strings.Contains(out, "empty value") || !strings.Contains(out, "nothing was restarted") {
+ t.Fatalf("%v = %q, want the empty-selector refusal", args, out)
+ }
+ }
+
+ // The selectors do not change the mode refusal: a narrowed restart is
+ // still a console verb, and direct bucket mode still says so in restart's
+ // own words.
+ cmd := exec.Command(bin, "restart", "-app", "titogo", "-host", "bill.go.tito.io")
+ cmd.Env = carlosTestEnv(t, "CARLOS_DEPLOYMENT_DIR="+t.TempDir())
+ cmd.Dir = scratch
+ raw, err := cmd.CombinedOutput()
+ if err == nil {
+ t.Fatalf("a narrowed restart in direct bucket mode exited 0:\n%s", raw)
+ }
+ if !strings.Contains(string(raw), "carlos restart works through a console") {
+ t.Fatalf("direct-mode refusal = %q, want restart's own message", raw)
+ }
+}