carlos / platform Public

cli, docs: carlos run takes an idempotency key, and the run role is documented (PRO-6214)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Paul Campbell pushed by claude 1 day ago 95bb8e9603af87bd5b8571e180e8627a22e14f8c parent 6dc07d3

13 files changed, +361 −24

auth.go +10 −0
docs/site/cli.md +72 −2
docs/superpowers/specs/2026-10-08-carlos-run-design.md +2 −0
internal/console/apiruns.go +16 −5
internal/console/apiruns_service_test.go +26 −2
internal/console/authstore.go +1 −1
internal/host/runevents.go +2 −2
internal/host/runevents_service_test.go +2 −2
internal/runs/commands.go +5 −0
internal/runs/runs.go +12 −2
run.go +56 −7
run_test.go +156 −0
services.go +1 −1

diff --git a/auth.go b/auth.go

index e0b29a9d8..47565dc94 100644

--- a/auth.go

+++ b/auth.go

@@ -494,6 +494,13 @@ func (c *consoleClient) doMultipart(ctx context.Context, path string, write func

// is a 405 rather than an upload. Everything else — the pipe, the client and

// the raw non-2xx body — is doMultipart's, unchanged and not a second copy.

func (c *consoleClient) doMultipartMethod(ctx context.Context, method, path string, write func(*multipart.Writer) error) (int, []byte, error) {

+ return c.doMultipartHeaders(ctx, method, path, nil, write)

+// doMultipartHeaders is doMultipartMethod with extra request headers, such

+// as a run's Idempotency-Key. They are set before the content type and the

+// bearer, which they cannot override.

+func (c *consoleClient) doMultipartHeaders(ctx context.Context, method, path string, headers http.Header, write func(*multipart.Writer) error) (int, []byte, error) {

pr, pw := io.Pipe()

mw := multipart.NewWriter(pw)

go func() {

@@ -512,6 +519,9 @@ func (c *consoleClient) doMultipartMethod(ctx context.Context, method, path stri

pr.Close()

return 0, nil, err

}

+ for k, vs := range headers {

+ req.Header[k] = vs

+ }

req.Header.Set("Content-Type", mw.FormDataContentType())

if c.token != "" {

req.Header.Set("Authorization", "Bearer "+c.token)

diff --git a/docs/site/cli.md b/docs/site/cli.md

index 0a8e384f2..64ad50590 100644

--- a/docs/site/cli.md

+++ b/docs/site/cli.md

@@ -586,7 +586,7 @@ Runs one command of your app against one instance, as that instance: the same bu

carlos run --app shop --instance shop.example.com --file orders.csv -- import orders.csv

```

-Any person who is a member of the account can run a command. A service credential cannot.

+Any person who is a member of the account can run a command. A service credential cannot. The one exception is a `run` credential, which can run the commands it was minted with. [Runs from a service](#runs-from-a-service) says how.

Each --file is uploaded with the request and placed, under its own name, in the directory CARLOS_RUN_DIR names. Together they can come to 64 MB. The directory is deleted when the run ends.

@@ -596,10 +596,78 @@ A hibernating instance is woken first. A live one keeps serving, and the command

carlos run exits with the command's own status, 128 plus the signal number when a signal ended it, 124 when it timed out, 125 when it was interrupted or the instance let go of it without a result, 130 when you cancelled it, 3 when the box refused it or did not pick it up within 60 seconds, and 2 when the console refused it.

-carlos logs shows when each run started, who asked for it and how it ended. The command's output is not kept there.

+carlos logs shows when each run started, who asked for it and how it ended. The command's output is not kept there. A service's run reads "started by service ci", with the person it named, if any, marked as the service's statement.

+If you cannot tell whether a run started (the connection dropped as you sent it), start it with `--idempotency-key <key>`. The key is a string you choose: 1 to 128 characters of letters, digits, dots, underscores, colons and hyphens. Run the same command again with the same key and the same files, and carlos says `already started: run <id>` and follows that run, instead of starting a second. Each `--file` is hashed first, because a key only matches a request that is the same in every byte. The same key with a different command, timeout or file is refused with `idempotency_key_reused`. A key is yours alone: no other person or service can see it or collide with it. It is honoured for 24 hours, or until you have started 32 newer runs with keys on that instance, whichever comes first. After that the same key starts a new run.

Arguments are kept in the account's audit log, with who ran them. Never put a secret in an argument. Pass it in a --file, or keep it in the instance's environment.

+#### Runs from a service

+A server that has no person at a terminal, such as a back office that imports an account's data, can start and follow runs with a `run` credential. The account owner mints it:

+```sh

+carlos services create --role run --app shop --commands "import classic,import sync" fleet-runs

+```

+`--app` and `--commands` are both required. A `run` credential starts, follows, cancels, expires and acknowledges runs of its own app's instances, and does nothing else. No other role can run commands, `admin` included: a credential that can provision instances cannot run commands on them unless it was also minted for that.

+`--commands` is a comma-separated list of prefixes, and each prefix is its words separated by spaces. A run is allowed when its arguments start with every word of at least one prefix, compared word for word and exactly. With only `import classic` granted, `import classic --slug acme` runs, and `import classical`, `import` and `import sync` are refused. **A prefix also allows every flag and argument after it**, so grant the shortest prefix that names the work you mean, and do not grant `import` when you mean `import classic`. A prefix of no words is refused. A word is valid UTF-8 of 1 to 128 bytes with no spaces, commas or control characters, and a credential may hold up to 32 prefixes. A refused command answers 403 `command_not_granted` before anything is stored.

+The list is fixed at mint. `carlos services ls` shows it, rotating a credential keeps it, and there is no way to edit it: mint a new credential to change what it may run. People are not limited by any list.

+A service sees and acts on only the runs its own credential started. Another service's run, a person's run and a run of an earlier credential that had the same name all answer `not_found`, as if there were no such run. Members of the account see every run, a service's included. When a service's run holds an instance, a service that did not start it is told that the instance has a run open and nothing more.

+A service may say who a run is for, by sending `on_behalf_of` with an email address of at most 254 bytes. It is recorded on the service's word and checked for nothing but its shape, so it grants nothing. Every place a run is shown reads "run by service ci for ada@example.com (as stated by the service)", or "run by service ci (scheduled)" when it names no one. A person's run cannot send it.

+#### The run API

+`carlos run` is a client of these routes. A service calls them with its credential as a bearer token, and a person with a signed-in token. Under `/api/cli/apps/<account>/<app>/instances/<host>/runs`:

+- `POST` creates a run. The body is multipart: a `meta` part first, as JSON (`argv`, `timeout_s`, `files` as a list of `name`, `size` and `sha256`, and `on_behalf_of`), then one `file` part per file, in the order and under the names `meta` gave. It answers `202` with the run's `id`, `host` and `state`.

+- `GET /<id>?after=<seq>` pages the run's output from after chunk `seq`, and carries the result once the run has one.

+- `POST /<id>/cancel` and `POST /<id>/expire` cancel a run, or give up on one no box has picked up.

+- `POST /<id>/ack` with `{"last_seq": n}` says the output has been read, so the console can delete it.

+- `GET ?key=<key>` answers `{"run_id": "..."}` for the run a key started, or `404`.

+Every refusal a run route makes has a `code` beside its `error` sentence, for a caller that has no person to read the sentence. Not being signed in, a credential of the wrong role and an account the caller is not in are answered as they are on every other route, without a code, and tell themselves apart by status.

+- `not_found`: no such instance, or no such run. For a service, also a run it did not start.

+- `host_claim`: this console runs with host claims, which cannot yet run commands, so it refuses every run for everyone. The same refusal comes to a `run` credential.

+- `rate_limited`: too many run requests. Wait a minute.

+- `unavailable`: the console's store is not available. Try again.

+- `record_failed`: the console could not record the run or the change. Try again.

+- `read_failed`: the console could not read the run. Try again.

+- `not_runnable`: the instance is of a kind `carlos run` does not work on.

+- `moving`: the instance is moving. Run it once the move has finished.

+- `run_open`: the instance already has a run open. A service that did not start it is not told whose it is.

+- `malformed`: the request is not shaped as above.

+- `invalid_request`: the command, timeout, files or sha256 values in `meta` are not acceptable.

+- `files_too_large`: the files come to more than 64 MB.

+- `import_uncommitted`: the instance is an import that is not committed yet.

+- `command_not_granted`: this credential was not minted for that command.

+- `on_behalf_of_invalid`: `on_behalf_of` is not an email address of at most 254 bytes, or a person sent it.

+- `run_started`: the run has already started, so it cannot be expired.

+- `run_ended`: the run has already ended, so it cannot be cancelled.

+- `run_not_finished`: the run has not finished, so it cannot be acknowledged yet.

+- `last_seq_mismatch`: the acknowledgement named a chunk other than the run's last.

+- `idempotency_key_invalid`: the key is not 1 to 128 characters of letters, digits, dots, underscores, colons and hyphens.

+- `idempotency_key_reused`: the key was used for a different request.

+- `keys_not_ready`: this console does not take keys yet. See below.

+- `file_hash_mismatch`: the bytes of a file are not the sha256 `meta` declared for it.

+A create with an `Idempotency-Key` header is admitted at most once. A retry with the same key and the same request answers `202` with the same body and `"replayed": true`, and starts nothing. Keyed creates carry three rules:

+- `meta` must declare each file's sha256, in either case. A file whose bytes do not match is refused as `file_hash_mismatch`.

+- A refusal is not remembered, and a key is evaluated afresh on each try. A `run_open` that blocked the first try can succeed on the second, once the other run has finished.

+- A 4xx on a retry means that attempt admitted nothing. An earlier attempt with the same key may have started the run, so confirm with `GET ?key=` before concluding that it never ran. A 5xx, a timeout or a dropped connection means unknown: retry with the same key.

+A key is honoured for 24 hours, or until the same caller has made 32 newer keyed admissions on that instance, whichever comes first. The limit is per caller, so another caller can never evict your key. Do not retry a key past that: it starts a fresh run.

+Keyed creates are off until the operator of the console turns them on, with `CARLOS_RUN_KEYS=on`, and answer `503 keys_not_ready` until then. The operator does that only once every box runs a build that keeps them. It is a one-way switch: a box rolled back to an older build afterwards forgets the keys silently, and a retry could then start a second run.

+Deleting an instance deletes its keys with it. A retry that arrives after the host was deleted and created again starts a fresh run on the new instance.

### carlos schedule

Gives an app a timetable. A schedule is a time and a path: at each fire the

@@ -1868,6 +1936,8 @@ It must name one app and cannot hotfix. If the app has a pipeline, it may

move `canary/<name>` channels only. CI reads it from

`CARLOS_SERVICE_TOKEN` with `CARLOS_CONSOLE`.

+`run` is for a server that starts commands of an app against its instances, and no other role does that. It is minted with `--app` and `--commands`, and it can run only the commands it names. See [Runs from a service](#runs-from-a-service); it also says what a prefix allows.

One door stays shut to every role: `carlos instances enable`, which writes the

manifest bounding which domain, pool and regions an instance may be created

in — and, with `--private-storage`, whether each instance gets an object store

diff --git a/docs/superpowers/specs/2026-10-08-carlos-run-design.md b/docs/superpowers/specs/2026-10-08-carlos-run-design.md

index 6cc22fa02..30a6738c1 100644

--- a/docs/superpowers/specs/2026-10-08-carlos-run-design.md

+++ b/docs/superpowers/specs/2026-10-08-carlos-run-design.md

@@ -31,6 +31,8 @@ instance".

`carlos db pull`. Service credentials are refused. A member can already

read every instance's database and publish to non-stable channels, so

this grants nothing new.

+ *Amended 2026-10-24: a `run` service credential may, for the commands it

+ was granted. See `2026-10-24-service-runs-design.md`.*

2. **Output transport:** the run's own chunks in the deployment bucket,

polled by the CLI through the console. There is no new network path; the

console still never connects to a box.

diff --git a/internal/console/apiruns.go b/internal/console/apiruns.go

index db80bf6e1..02a91d28a 100644

--- a/internal/console/apiruns.go

+++ b/internal/console/apiruns.go

@@ -139,8 +139,15 @@ const claimRunUnsupportedMsg = "carlos run is not available on this console yet.

const runTooManyMsg = "too many run requests. wait a minute and try again"

-// runOpenMsg is refusal 7 (RUN-C24), at the door and at admission.

-func runOpenMsg(host string, run provision.RunState) string {

+// runOpenMsg is refusal 7 (RUN-C24), at the door and at admission. req is

+// the open run's request when it could be read: a service's run is named in

+// the house line (runs.Request.ServiceLine), never by its audit address.

+func runOpenMsg(host string, run provision.RunState, req *runs.Request) string {

+ if req != nil {

+ if line := req.ServiceLine(); line != "" {

+ return fmt.Sprintf("%s already has a run open: %s, %s", host, run.ID, line)

+ }

return fmt.Sprintf("%s already has a run open: %s, started by %s", host, run.ID, run.By)

}

@@ -154,11 +161,15 @@ func runOpenRedactedMsg(host string) string {

// run's id and requester only when the run is its own; a run whose request

// cannot be read is not shown to be its own.

func (s *Server) runOpenRefusal(ctx context.Context, p Principal, sqid, app, host string, run provision.RunState) string {

+ req, err := runs.ReadRequest(ctx, s.cfg.Releases, sqid, app, run.ID)

+ if err != nil {

+ req = nil

+ }

if p.Kind == PrincipalUser {

- return runOpenMsg(host, run)

+ return runOpenMsg(host, run, req)

}

- if req, err := runs.ReadRequest(ctx, s.cfg.Releases, sqid, app, run.ID); err == nil && s.runVisibleTo(p, *req) {

- return runOpenMsg(host, run)

+ if req != nil && s.runVisibleTo(p, *req) {

+ return runOpenMsg(host, run, req)

}

return runOpenRedactedMsg(host)

}

diff --git a/internal/console/apiruns_service_test.go b/internal/console/apiruns_service_test.go

index 7e7f91fe4..eb58715d0 100644

--- a/internal/console/apiruns_service_test.go

+++ b/internal/console/apiruns_service_test.go

@@ -11,6 +11,7 @@ import (

"go/parser"

"go/token"

"net/http"

+ "os"

"slices"

"strings"

"sync"

@@ -354,9 +355,10 @@ func TestRunOpenRefusalRedactedForServices(t *testing.T) {

if status != http.StatusConflict || codeOf(t, body) != "run_open" || strings.Contains(body, own) || strings.Contains(body, "service:") {

t.Fatalf("service create over another service's run: %d %s", status, body)

}

- // The member is told whose it is, as the service it was.

+ // The member is told whose it is, in the house line: the service, and

+ // the person only as the service's statement.

status, body = postRun(t, fx.runsURL(runTestHost), fx.member, okMeta(), nil)

- if want := runTestHost + " already has a run open: " + own + ", started by service:" + release.OpsAccount + "/ci"; status != http.StatusConflict || errorOf(t, body) != want {

+ if want := runTestHost + " already has a run open: " + own + ", run by service ci for " + runServiceFor + " (as stated by the service)"; status != http.StatusConflict || errorOf(t, body) != want {

t.Fatalf("member over a service's run: %d %s, want %q", status, body, want)

}

@@ -847,3 +849,25 @@ func TestRunGrantedServiceOnAHostClaimConsoleRefused(t *testing.T) {

t.Errorf("a granted create on a host-claim console = %d %s", status, body)

}

+// Every refusal code a run handler writes is listed in the run API section

+// of the CLI page: a caller with no person to read the sentence acts on the

+// code, so a code the docs do not name is a code nobody can act on.

+func TestRunRefusalCodesAreDocumented(t *testing.T) {

+ raw, err := os.ReadFile("../../docs/site/cli.md")

+ if err != nil {

+ t.Fatal(err)

+ }

+ _, section, ok := strings.Cut(string(raw), "\n#### The run API\n")

+ if !ok {

+ t.Fatal("docs/site/cli.md has no '#### The run API' section")

+ }

+ if i := strings.Index(section, "\n#"); i >= 0 {

+ section = section[:i]

+ }

+ for _, code := range runRefusalCodes {

+ if !strings.Contains(section, "- `"+code+"`:") {

+ t.Errorf("the run API docs do not list the code %s", code)

+ }

diff --git a/internal/console/authstore.go b/internal/console/authstore.go

index 942ecbecb..ed50fd768 100644

--- a/internal/console/authstore.go

+++ b/internal/console/authstore.go

@@ -159,7 +159,7 @@ func (t CLIToken) CommandsUnreadable() bool {

}

// UnreadableCommands is what a listing shows for CommandsUnreadable.

-const UnreadableCommands = "(unreadable — refused at sign-in)"

+const UnreadableCommands = runs.UnreadableCommands

// CommandList is Commands for a person to read: "import classic, import

// sync", or UnreadableCommands for a run row whose list cannot be read.

diff --git a/internal/host/runevents.go b/internal/host/runevents.go

index 71d554ada..d20cbb054 100644

--- a/internal/host/runevents.go

+++ b/internal/host/runevents.go

@@ -24,8 +24,8 @@ const runEventLevel = applog.LevelInfo

// run names the service, and the person it named only as its statement

// (service runs spec §2).

func runStartedLine(req runs.Request) string {

- if line := req.ServiceLine(); line != "" {

- return fmt.Sprintf("run %s started: %s", req.ID, line)

+ if by := req.ServiceBy(); by != "" {

+ return fmt.Sprintf("run %s started %s", req.ID, by)

}

return fmt.Sprintf("run %s started by %s", req.ID, req.By)

}

diff --git a/internal/host/runevents_service_test.go b/internal/host/runevents_service_test.go

index c88842c31..7b92f3475 100644

--- a/internal/host/runevents_service_test.go

+++ b/internal/host/runevents_service_test.go

@@ -14,7 +14,7 @@ func TestRunStartedLineForAServicesRun(t *testing.T) {

req := runs.Request{ID: "r1", By: "service:bab/ci", ByService: "ci", ByCredential: "c0ffee", OnBehalfOf: "paul@example.com",

Argv: []string{"import", "classic", "--token", "s3cret"}}

got := runStartedLine(req)

- if want := "run r1 started: run by service ci for paul@example.com (as stated by the service)"; got != want {

+ if want := "run r1 started by service ci for paul@example.com (as stated by the service)"; got != want {

t.Fatalf("got %q, want %q", got, want)

}

if strings.Contains(got, "by paul@example.com") || strings.Contains(got, "c0ffee") {

@@ -26,7 +26,7 @@ func TestRunStartedLineForAServicesRun(t *testing.T) {

}

req.OnBehalfOf = ""

- if got, want := runStartedLine(req), "run r1 started: run by service ci (scheduled)"; got != want {

+ if got, want := runStartedLine(req), "run r1 started by service ci (scheduled)"; got != want {

t.Fatalf("scheduled: got %q, want %q", got, want)

}

diff --git a/internal/runs/commands.go b/internal/runs/commands.go

index a9941da2f..3691d080d 100644

--- a/internal/runs/commands.go

+++ b/internal/runs/commands.go

@@ -118,6 +118,11 @@ func ParseCommandList(s string) [][]string {

return out

}

+// UnreadableCommands is what a listing shows, in place of a command list,

+// for a run credential whose stored list cannot be read. The console and the

+// CLI both print it.

+const UnreadableCommands = "(unreadable — refused at sign-in)"

// FormatCommandList is a command list for a person to read:

// "import classic, import sync".

func FormatCommandList(prefixes [][]string) string {

diff --git a/internal/runs/runs.go b/internal/runs/runs.go

index f0e68b526..227df8a47 100644

--- a/internal/runs/runs.go

+++ b/internal/runs/runs.go

@@ -204,13 +204,23 @@ func (r Request) Timeout() time.Duration { return time.Duration(r.TimeoutS) * ti

// "" for a person's. The person a service names is always shown as that

// service's statement, never as the one who ran it.

func (r Request) ServiceLine() string {

+ if by := r.ServiceBy(); by != "" {

+ return "run " + by

+ }

+ return ""

+// ServiceBy is ServiceLine without its leading "run ", for a sentence that

+// already says "run" ("run <id> started by service ci (scheduled)"). "" for

+// a person's run.

+func (r Request) ServiceBy() string {

switch {

case r.ByService == "":

return ""

case r.OnBehalfOf == "":

- return "run by service " + r.ByService + " (scheduled)"

+ return "by service " + r.ByService + " (scheduled)"

}

- return "run by service " + r.ByService + " for " + r.OnBehalfOf + " (as stated by the service)"

+ return "by service " + r.ByService + " for " + r.OnBehalfOf + " (as stated by the service)"

}

// Result is runs/<id>/result.json (run spec §5.5), written only after the

diff --git a/run.go b/run.go

index 645035006..01058e491 100644

--- a/run.go

+++ b/run.go

@@ -2,6 +2,8 @@ package main

import (

"context"

+ "crypto/sha256"

+ "encoding/hex"

"encoding/json"

"errors"

"flag"

@@ -29,7 +31,11 @@ const runUsageRun = "usage: carlos run --app <app> --instance <host> [--file <pa

const runUsageFollow = "carlos run --app <app> --instance <host> --follow <id>"

-func runUsageText() string { return runUsageRun + "\n " + runUsageFollow + "\n" }

+const runUsageKey = "add --idempotency-key <key> to start a run you can safely run again: the same key and request follow the run already started"

+func runUsageText() string {

+ return runUsageRun + "\n " + runUsageFollow + "\n " + runUsageKey + "\n"

const runDirectModeMsg = "carlos run works through a console, and this shell is in direct bucket mode. unset CARLOS_DEPLOYMENT_BUCKET and CARLOS_DEPLOYMENT_DIR, then run carlos auth login --console <url>"

@@ -40,6 +46,7 @@ var errRunUsage = errors.New("carlos run: usage")

// runOpts is `carlos run` parsed.

type runOpts struct {

app, instance, follow, console string

+ idempotencyKey string

acctRaw, acctIDRaw *string

files []string

timeout time.Duration

@@ -63,6 +70,7 @@ func parseRunArgs(args []string, stderr io.Writer) (runOpts, error) {

fs.Var(&files, "file", "a file to upload with the run (repeatable; 64 MB in total)")

timeout := fs.Duration("timeout", runs.DefaultTimeout, "how long the command may run (1s to 6h)")

follow := fs.String("follow", "", "attach to a run already started, by its id")

+ key := fs.String("idempotency-key", "", "a key of your choosing: run again with the same key and the same request, and you follow the run already started instead of starting a second")

acctRaw, acctIDRaw, _ := accountFlags(fs)

console := consoleOption(fs, "console to run through (default: the only logged-in one)")

fs.Usage = flagUsage(fs, runUsageText())

@@ -73,7 +81,8 @@ func parseRunArgs(args []string, stderr io.Writer) (runOpts, error) {

return runOpts{}, errRunUsage

}

o := runOpts{app: strings.TrimSpace(*app), instance: strings.TrimSpace(*instance), follow: strings.TrimSpace(*follow),

- console: *console, acctRaw: acctRaw, acctIDRaw: acctIDRaw, files: files, timeout: *timeout, argv: argv}

+ console: *console, acctRaw: acctRaw, acctIDRaw: acctIDRaw, files: files, timeout: *timeout, argv: argv,

+ idempotencyKey: *key}

switch {

case o.instance == "":

return o, errors.New("--app and --instance name the instance to run against")

@@ -81,10 +90,17 @@ func parseRunArgs(args []string, stderr io.Writer) (runOpts, error) {

return o, errors.New("--follow takes a run id and no command after --")

case runs.CheckTimeout(o.timeout) != nil:

return o, errors.New("--timeout must be between 1s and 6h")

+ case o.idempotencyKey != "" && !runs.ValidKey(o.idempotencyKey):

+ return o, errors.New(runKeyShapeMsg)

+ case o.idempotencyKey != "" && o.follow != "":

+ return o, errors.New("--idempotency-key is for starting a run. --follow starts none")

}

return o, nil

}

+// runKeyShapeMsg is what the CLI says of a key that the console would refuse.

+const runKeyShapeMsg = "--idempotency-key is 1 to 128 characters: letters, digits, dots, underscores, colons and hyphens"

// errRunNotRegular is a --file that is a directory or a device: only a

// regular file has a size to promise the console.

var errRunNotRegular = errors.New("not a regular file")

@@ -111,7 +127,9 @@ func runReadError(path string, err error) error {

}

// runMetaFor is the create request's meta: argv, the timeout, and each

-// file's base name and size. The console checks all of it again.

+// file's base name and size. A run with an idempotency key also declares

+// each file's sha256: the console needs it to tell a retry of the same

+// request from a different one. The console checks all of it again.

func runMetaFor(o runOpts) (runs.Meta, error) {

m := runs.Meta{Argv: o.argv, TimeoutS: int64(o.timeout / time.Second)}

for _, p := range o.files {

@@ -122,11 +140,31 @@ func runMetaFor(o runOpts) (runs.Meta, error) {

if !st.Mode().IsRegular() {

return m, runReadError(p, errRunNotRegular)

}

- m.Files = append(m.Files, runs.MetaFile{Name: filepath.Base(p), Size: st.Size()})

+ f := runs.MetaFile{Name: filepath.Base(p), Size: st.Size()}

+ if o.idempotencyKey != "" {

+ if f.SHA256, err = runFileSum(p); err != nil {

+ return m, runReadError(p, err)

+ }

+ m.Files = append(m.Files, f)

}

return m, nil

}

+// runFileSum is the hex sha256 of the file at path.

+func runFileSum(path string) (string, error) {

+ f, err := os.Open(path)

+ if err != nil {

+ return "", err

+ }

+ defer f.Close()

+ h := sha256.New()

+ if _, err := io.Copy(h, f); err != nil {

+ return "", err

+ }

+ return hex.EncodeToString(h.Sum(nil)), nil

// runClient is the console's five run routes (run spec §5.1, §5.6, §5.7).

type runClient interface {

create(ctx context.Context, meta runs.Meta, paths []string) (runs.Created, error)

@@ -171,6 +209,7 @@ type runAPI struct {

a apiAcct

app string

host string

+ key string // the Idempotency-Key a create carries; "" sends none

}

func (c runAPI) path(tail string) string {

@@ -188,7 +227,11 @@ func (c runAPI) create(ctx context.Context, meta runs.Meta, paths []string) (run

}

var mu sync.Mutex

var writeErr error

- status, body, err := c.s.c.doMultipart(ctx, c.path(""), func(mw *multipart.Writer) (err error) {

+ var hdr http.Header

+ if c.key != "" {

+ hdr = http.Header{"Idempotency-Key": {c.key}}

+ }

+ status, body, err := c.s.c.doMultipartHeaders(ctx, http.MethodPost, c.path(""), hdr, func(mw *multipart.Writer) (err error) {

defer func() {

mu.Lock()

writeErr = err

@@ -749,7 +792,13 @@ func startRun(ctx context.Context, c runClient, t runTerm, o runOpts, app string

t.say(fmt.Sprintf("carlos run: %s", err))

return 1

}

- t.say(fmt.Sprintf("run %s on %s. to follow it again: carlos run --app %s --instance %s --follow %s", created.ID, o.instance, app, o.instance, created.ID))

+ if created.Replayed {

+ // An earlier attempt with this key started the run: this one starts

+ // nothing, and follows it.

+ t.say(fmt.Sprintf("already started: run %s on %s. to follow it again: carlos run --app %s --instance %s --follow %s", created.ID, o.instance, app, o.instance, created.ID))

+ } else {

+ t.say(fmt.Sprintf("run %s on %s. to follow it again: carlos run --app %s --instance %s --follow %s", created.ID, o.instance, app, o.instance, created.ID))

+ }

return followRun(ctx, c, t, o.instance, created.ID, t.now().Add(runs.PickupWindow))

}

@@ -779,7 +828,7 @@ func runRun(args []string) {

interrupts := make(chan os.Signal, 2)

signal.Notify(interrupts, os.Interrupt, syscall.SIGTERM)

t := runTerm{stdout: os.Stdout, stderr: os.Stderr, interrupts: interrupts, after: time.After, now: time.Now}

- c := runAPI{s: sess, a: acct, app: app, host: o.instance}

+ c := runAPI{s: sess, a: acct, app: app, host: o.instance, key: o.idempotencyKey}

ctx := context.Background()

if o.follow != "" {

os.Exit(followRun(ctx, c, t, o.instance, o.follow, time.Time{}))

diff --git a/run_test.go b/run_test.go

index a1d797383..3c76f2463 100644

--- a/run_test.go

+++ b/run_test.go

@@ -3,6 +3,8 @@ package main

import (

"bytes"

"context"

+ "crypto/sha256"

+ "encoding/hex"

"encoding/json"

"errors"

"io"

@@ -13,6 +15,7 @@ import (

"os"

"path/filepath"

"slices"

+ "strconv"

"strings"

"sync"

"sync/atomic"

@@ -1265,3 +1268,156 @@ func TestStartRunFailures(t *testing.T) {

}

+// A keyed run sends the key as the Idempotency-Key header, and declares each

+// file's sha256 in the meta so the console can tell a retry of the same

+// request from a different one under the same key.

+func TestRunIdempotencyKeyFlag(t *testing.T) {

+ o, err := parseRunArgs([]string{"--instance", "h.example.com", "--idempotency-key", "import-42:a.b_c", "--", "import"}, io.Discard)

+ if err != nil || o.idempotencyKey != "import-42:a.b_c" {

+ t.Fatalf("opts %+v, %v", o, err)

+ }

+ dir := t.TempDir()

+ p := filepath.Join(dir, "x.csv")

+ os.WriteFile(p, []byte("1,2"), 0o600)

+ sum := sha256.Sum256([]byte("1,2"))

+ want := hex.EncodeToString(sum[:])

+ m, err := runMetaFor(runOpts{argv: []string{"import"}, timeout: time.Minute, files: []string{p}, idempotencyKey: "k1"})

+ if err != nil || len(m.Files) != 1 || m.Files[0].SHA256 != want {

+ t.Fatalf("keyed meta %+v, %v, want sha256 %s", m, err, want)

+ }

+ m, err = runMetaFor(runOpts{argv: []string{"import"}, timeout: time.Minute, files: []string{p}})

+ if err != nil || m.Files[0].SHA256 != "" {

+ t.Fatalf("an unkeyed run declares no hash: %+v, %v", m, err)

+ }

+ var header, metaJSON string

+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ header = r.Header.Get("Idempotency-Key")

+ mr, err := r.MultipartReader()

+ if err != nil {

+ t.Error(err)

+ return

+ }

+ for {

+ part, err := mr.NextPart()

+ if err != nil {

+ break

+ }

+ raw, _ := io.ReadAll(part)

+ if part.FormName() == "meta" {

+ metaJSON = string(raw)

+ }

+ w.WriteHeader(http.StatusAccepted)

+ json.NewEncoder(w).Encode(runs.Created{ID: "run_a", Host: "h.example.com", State: "queued"})

+ }))

+ defer ts.Close()

+ c := runAPI{s: newTestSession(ts.URL), a: apiAcct{Seg: "acme"}, app: "shop", host: "h.example.com", key: "k1"}

+ meta, err := runMetaFor(runOpts{argv: []string{"import"}, timeout: time.Minute, files: []string{p}, idempotencyKey: "k1"})

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, err := c.create(context.Background(), meta, []string{p}); err != nil {

+ t.Fatal(err)

+ }

+ if header != "k1" || !strings.Contains(metaJSON, `"sha256":"`+want+`"`) {

+ t.Fatalf("header %q, meta %s", header, metaJSON)

+ }

+ c.key = ""

+ header = "unset"

+ if _, err := c.create(context.Background(), meta, []string{p}); err != nil || header != "" {

+ t.Fatalf("an unkeyed create sent the header %q, %v", header, err)

+ }

+// An invalid key is refused by the CLI, before any network call: runRun

+// parses its arguments before it resolves a console.

+func TestRunIdempotencyKeyIsCheckedBeforeAnyCall(t *testing.T) {

+ for name, key := range map[string]string{

+ "a space": "two words",

+ "a slash": "a/b",

+ "too long": strings.Repeat("k", 129),

+ "non-ascii": "klé",

+ } {

+ _, err := parseRunArgs([]string{"--instance", "h.example.com", "--idempotency-key", key, "--", "x"}, io.Discard)

+ if err == nil || !strings.Contains(err.Error(), "--idempotency-key") {

+ t.Errorf("%s: %v, want a refusal naming --idempotency-key", name, err)

+ }

+ // An empty value is the same as not passing the flag.

+ if o, err := parseRunArgs([]string{"--instance", "h.example.com", "--idempotency-key", "", "--", "x"}, io.Discard); err != nil || o.idempotencyKey != "" {

+ t.Errorf("empty key: %+v, %v", o, err)

+ }

+ // A key belongs to starting a run; --follow starts none.

+ if _, err := parseRunArgs([]string{"--instance", "h.example.com", "--follow", "run_x", "--idempotency-key", "k"}, io.Discard); err == nil {

+ t.Error("--follow with a key accepted")

+ }

+// keyedConsole is a console that admits a keyed create once: the second

+// create under the same key answers 202 with the same run and replayed.

+type keyedConsole struct {

+ mu sync.Mutex

+ creates int // create requests made

+ started int // runs started

+ admitted map[string]string

+ pages []string // run ids paged

+func (k *keyedConsole) ServeHTTP(w http.ResponseWriter, r *http.Request) {

+ k.mu.Lock()

+ defer k.mu.Unlock()

+ if r.Method == http.MethodPost && strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/") {

+ k.creates++

+ key := r.Header.Get("Idempotency-Key")

+ if id, ok := k.admitted[key]; ok {

+ w.WriteHeader(http.StatusAccepted)

+ json.NewEncoder(w).Encode(runs.Created{ID: id, Host: "h.example.com", State: "done", Replayed: true})

+ return

+ }

+ k.started++

+ id := "run_" + strconv.Itoa(k.started)

+ k.admitted[key] = id

+ w.WriteHeader(http.StatusAccepted)

+ json.NewEncoder(w).Encode(runs.Created{ID: id, Host: "h.example.com", State: "queued"})

+ return

+ }

+ if r.Method == http.MethodGet {

+ k.pages = append(k.pages, r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:])

+ json.NewEncoder(w).Encode(runs.Page{State: "done", Result: &runs.Result{Exit: 0}})

+ return

+ }

+ json.NewEncoder(w).Encode(runs.StateAnswer{State: "done"}) // ack

+func TestRunCLIRetryFollowsTheReplayedRun(t *testing.T) {

+ k := &keyedConsole{admitted: map[string]string{}}

+ ts := httptest.NewServer(k)

+ defer ts.Close()

+ o := runOpts{instance: "h.example.com", argv: []string{"import"}, timeout: time.Minute, idempotencyKey: "import-42"}

+ c := runAPI{s: newTestSession(ts.URL), a: apiAcct{Seg: "acme"}, app: "shop", host: "h.example.com", key: o.idempotencyKey}

+ first := newTermFx()

+ if code := startRun(context.Background(), c, first.term(), o, "shop"); code != 0 {

+ t.Fatalf("first: exit %d, stderr %q", code, first.err.String())

+ }

+ if strings.Contains(first.err.String(), "already started") || !strings.Contains(first.err.String(), "run run_1 on h.example.com. to follow it again") {

+ t.Fatalf("a fresh run says %q", first.err.String())

+ }

+ second := newTermFx()

+ if code := startRun(context.Background(), c, second.term(), o, "shop"); code != 0 {

+ t.Fatalf("retry: exit %d, stderr %q", code, second.err.String())

+ }

+ if !strings.Contains(second.err.String(), "already started: run run_1 on h.example.com. to follow it again: carlos run --app shop --instance h.example.com --follow run_1") {

+ t.Fatalf("the retry says %q", second.err.String())

+ }

+ if k.creates != 2 || k.started != 1 {

+ t.Fatalf("%d creates, %d runs started: want 2 and 1", k.creates, k.started)

+ }

+ if want := []string{"run_1", "run_1"}; !slices.Equal(k.pages, want) {

+ t.Fatalf("followed %v, want %v: the retry follows the same run", k.pages, want)

+ }

diff --git a/services.go b/services.go

index d87147027..d015aa938 100644

--- a/services.go

+++ b/services.go

@@ -344,7 +344,7 @@ func runServicesList(args []string) {

cmds := strings.Join(commands, ",")

switch {

case r.CommandsUnreadable:

- cmds = "(unreadable — refused at sign-in)"

+ cmds = runs.UnreadableCommands

case cmds == "":

cmds = "-"

}