carlos / platform Public

cli, docs: carlos instances enable --private-storage (issue #412)

The flag reads through flagWasSet for the same absent-vs-explicit split
--channel and --seed have: omitting it preserves whatever policy the
manifest carries, and --private-storage=false turns it off. A `carlos
instances enable -channel beta` that silently unset a tenant's storage
isolation is the failure the wire test pins.

What `enable` prints is the door's ECHO, not the flag that was typed:
an enable that never mentioned storage still reports what the app does,
because that is the fact the next create acts on. `create` prints the
bucket it minted, which is the only place a member ever learns its
name.

Store mode takes the flag too, and says what the console would have
checked — that path writes the manifest straight to the bucket, so
nothing there verifies the deployment has object storage or that the
account is approved.

docs/site/cli.md gets the section, including the three things to know
before turning it on and the plain statement that nothing is ever torn
down automatically: deleting an instance leaves its bucket, exactly as
it already leaves the parked database.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

James Adam pushed by james@keymail.dev 1 month ago 276556fa0db91dc639f63fe932ed0450f75ad66e parent 91d16d0

6 files changed, +189 −16

apimode.go +19 −4
canary.go +1 −1
deploy.go +1 −1
docs/site/cli.md +48 −2
instances_cli.go +37 −8
instances_privatestorage_test.go +83 −0

diff --git a/apimode.go b/apimode.go

index 737cbe2e7..a1d9574d8 100644

--- a/apimode.go

+++ b/apimode.go

@@ -2769,6 +2769,12 @@ type apiInstanceManifest struct {

Channel string `json:"channel"`

Regions []string `json:"regions"`

Seed []string `json:"seed"`

+ // PrivateStorage is the app's standing policy that every instance it

+ // creates gets a config environment of its own and an object store

+ // bound to it (issue #412). Echoed by the door so `carlos instances

+ // enable` can print the policy in force rather than the one just

+ // asked for.

+ PrivateStorage bool `json:"private_storage"`

}

// enableError carries the instances-manifest door's STATUS alongside its

@@ -2811,11 +2817,12 @@ func enableUnavailable(err error) bool {

// pointers for the same reason apiConfigApply's ExecDelivery is: absent

// means "preserve what the manifest carries", and seed's explicit empty

// slice means "clear" — three requests the wire must keep apart.

-func (s *apiSession) instancesEnable(ctx context.Context, a apiAcct, app string, channel *string, seed *[]string) (apiInstanceManifest, error) {

+func (s *apiSession) instancesEnable(ctx context.Context, a apiAcct, app string, channel *string, seed *[]string, privateStorage *bool) (apiInstanceManifest, error) {

in := struct {

- Channel *string `json:"channel,omitempty"`

- Seed *[]string `json:"seed,omitempty"`

- }{channel, seed}

+ Channel *string `json:"channel,omitempty"`

+ Seed *[]string `json:"seed,omitempty"`

+ PrivateStorage *bool `json:"private_storage,omitempty"`

+ }{channel, seed, privateStorage}

status, body, err := s.putJSON(ctx, apiPath(a, app, "/instances-manifest"), in)

if err != nil {

return apiInstanceManifest{}, err

@@ -2898,6 +2905,14 @@ type apiInstanceRecord struct {

// measured it (instanceListRecord.DBBytes). nil when unknown, or from a

// console that predates the field — the DB column prints a dash.

DBBytes *int64 `json:"db_bytes,omitempty"`

+ // PrivateStoreBucket is the bucket the CREATE door provisioned for this

+ // instance under the app's private-storage policy (issue #412). Set on

+ // exactly one response — the create that minted it — and empty

+ // everywhere else, including on a list, because it is a fact about what

+ // one request did rather than a column of the record. What names the

+ // store afterwards is Environment, and `carlos store status` is what

+ // reads it.

+ PrivateStoreBucket string `json:"private_store_bucket"`

}

// apiUpstreamState mirrors status.UpstreamState's wire shape (internal/status)

diff --git a/canary.go b/canary.go

index 30c1309ce..da7f63e9c 100644

--- a/canary.go

+++ b/canary.go

@@ -378,7 +378,7 @@ func runCanaryDeploy(args []string) {

// instancesEnable is idempotent and is the ONLY source of the domain

// suffix — nothing here composes an apps domain by hand.

- m, err := sess.instancesEnable(ctx, acct, app2, nil, nil)

+ m, err := sess.instancesEnable(ctx, acct, app2, nil, nil, nil)

if err != nil {

log.Fatalf("canary: enable instances for %s: %v", app2, err)

}

diff --git a/deploy.go b/deploy.go

index c13a0ace3..67079baec 100644

--- a/deploy.go

+++ b/deploy.go

@@ -848,7 +848,7 @@ var errNoStaticHost = errors.New("no static host")

// about to promote to, and a record minted on a different one would serve

// something else.

func mintPlatformHost(ctx context.Context, sess *apiSession, acct apiAcct, app, channel string, messages ...string) (string, error) {

- m, err := sess.instancesEnable(ctx, acct, app, nil, nil)

+ m, err := sess.instancesEnable(ctx, acct, app, nil, nil, nil)

if err != nil {

return "", err

}

diff --git a/docs/site/cli.md b/docs/site/cli.md

index 3df903a7b..b1a32165e 100644

--- a/docs/site/cli.md

+++ b/docs/site/cli.md

@@ -580,6 +580,50 @@ bind it here, then restart the instance to pick the values up.

carlos instances set-environment --app jam --host new-queue.jam.bab.oncarlos.com --environment jam-canary

```

+#### Storage of its own, for every instance

+`--private-storage` on `enable` is one declaration that changes what every

+later `create` does. The app says, once, that each of its instances gets a

+config environment of its own with an object store bound to it — and from

+then on `carlos instances create` derives the environment from the host's

+first label, provisions that environment's bucket and IAM user, and delivers

+the coordinates into the bundle the new instance reads. No `store create`, no

+operator, no second command.

+```sh

+carlos instances enable --app jam --private-storage

+carlos instances create --app jam --host acme.jam.bab.oncarlos.com

+# created instance acme.jam.bab.oncarlos.com (channel stable, environment acme)

+# private storage: bucket carlos-bab-jam-acme

+```

+This is the shape for a managed product with one instance per customer, and

+the split is the point: declaring the policy is a signed-in person's act that

+no service credential can perform, while spending it is the ordinary create

+door an `instance`-role credential already reaches. So a public signup path

+provisions a tenant's bucket without ever holding the power to create one.

+Three things to know before you turn it on:

+- The account needs an **approved storage entitlement** (`carlos store

+ apply`) and the deployment needs object storage configured. `enable`

+ refuses the policy rather than letting it fail once per signup.

+- The host's first label becomes the environment name, so it must be a legal

+ one: lowercase letters, digits and hyphens, at most 32 characters, and not

+ `default`. A label that cannot name a store is refused at `create`, before

+ anything is written.

+- **Nothing is ever torn down automatically.** Deleting an instance leaves

+ its bucket, exactly as it already leaves the parked database. `carlos store

+ status --app <a>` lists every store the app has, which is where an

+ abandoned one is visible; destroying it is the console's own action, behind

+ the typed bucket name and a passkey.

+An app may have at most 100 environment stores by default; a deployment sets

+its own limit with `CARLOS_STORE_MAX_PER_APP`.

+`--private-storage=false` turns the policy off. Instances created before it

+keep the bindings and buckets they already have.

Typed on a box with no sub-verb, `carlos instances` does something different:

it lists that box's own registry routes, backing and owning unit included.

Off the box, [carlos routes](/docs/cli#carlos-routes) answers the same

@@ -1262,8 +1306,10 @@ one app, which also means it cannot create apps.

One door stays shut to every role: `carlos instances enable`, which writes the

manifest bounding which domain, pool and regions an instance may be created

-in. A person enables the app once and the credential provisions inside those

-bounds afterwards.

+in — and, with `--private-storage`, whether each instance gets an object store

+of its own. A person enables the app once and the credential provisions inside

+those bounds afterwards, including minting a tenant's bucket it could never

+have created directly.

The secret prints once, same as a fleet token, and the same distinction

applies at the other end: `rotate` refuses the credential's next call with

diff --git a/instances_cli.go b/instances_cli.go

index 5ffc16865..fe631ed67 100644

--- a/instances_cli.go

+++ b/instances_cli.go

@@ -59,6 +59,11 @@ func runInstancesEnable(args []string) {

channel := channelFlag(fs, "release channel stamped into every record at create (store-mode default stable; signed-in mode: omitted preserves the manifest's)")

regions := fs.String("regions", "", "comma-separated region allow-list (store mode only); empty = any shape-valid region")

seed := fs.String("seed", "", "argv of the app's first-boot init command, whitespace-split (strings.Fields)")

+ // A bool flag rather than a string, read through flagWasSet for the

+ // same absent-vs-explicit split --channel and --seed use: omitting it

+ // preserves whatever policy the manifest already carries, and

+ // --private-storage=false turns the policy off.

+ privateStorage := fs.Bool("private-storage", false, "give every instance of this app a config environment and an object store of its own (issue #412); --private-storage=false turns the policy off")

consoleFlag := consoleOption(fs, "console to enable through (signed-in mode only; default: the only logged-in one)")

acctRaw, acctIDRaw, acctOf := accountFlags(fs)

fs.Usage = flagUsage(fs, instancesEnableUsage)

@@ -83,6 +88,10 @@ func runInstancesEnable(args []string) {

if flagWasSet(fs, "channel") {

channelReq = channel

}

+ var privateStorageReq *bool

+ if flagWasSet(fs, "private-storage") {

+ privateStorageReq = privateStorage

+ }

var seedReq *[]string

if flagWasSet(fs, "seed") {

// parseSeed("") is an empty (non-nil after this) argv: an explicit

@@ -97,11 +106,17 @@ func runInstancesEnable(args []string) {

printUsing(stdout, sess)

acct := resolvedAccount(sess, acctRaw, acctIDRaw)

app2 := resolvedApp(sess, acct, *app, instancesEnableUsage)

- m, err := sess.instancesEnable(context.Background(), acct, app2, channelReq, seedReq)

+ m, err := sess.instancesEnable(context.Background(), acct, app2, channelReq, seedReq, privateStorageReq)

if err != nil {

log.Fatalf("instances enable: %v", err)

}

fmt.Fprintf(stdout, "%s for %s under %s\n", colorFor(stdout).ok("enabled instances"), m.App, m.Domain)

+ // The policy IN FORCE, from the door's echo, not the flag that was

+ // typed: an enable that never mentioned storage still prints what

+ // the app does, which is the fact the next create will act on.

+ if m.PrivateStorage {

+ mutedf(stdout, " every instance gets its own config environment and object store\n")

+ }

mutedf(stdout, " first instance: carlos instances create --app %s --host <label>.%s\n", m.App, m.Domain)

return

}

@@ -148,18 +163,29 @@ func runInstancesEnable(args []string) {

}

m := provision.Manifest{

- Schema: 1,

- App: *app,

- Domain: *domain,

- Pool: *pool,

- Channel: *channel,

- Regions: regionList,

- Seed: parseSeed(*seed),

+ Schema: 1,

+ App: *app,

+ Domain: *domain,

+ Pool: *pool,

+ Channel: *channel,

+ Regions: regionList,

+ Seed: parseSeed(*seed),

+ PrivateStorage: *privateStorage,

}

if err := instancesEnable(context.Background(), openStore(), acctOf(), *app, m); err != nil {

log.Fatal(err)

}

fmt.Printf("%s for %s under %s\n", colorFor(os.Stdout).ok("enabled instances"), *app, *domain)

+ if *privateStorage {

+ // Store mode writes the manifest straight to the bucket, so nothing

+ // checked that this deployment HAS object storage or that the

+ // account is approved for it — the console's enable door does both

+ // and refuses, but the operator path has no console to ask. Named

+ // here so the policy is not discovered to be inert at the first

+ // signup.

+ mutedf(os.Stdout, " every instance gets its own config environment and object store\n")

+ mutedf(os.Stdout, " the console minting them needs CARLOS_STORE_BACKEND and an approved store entitlement for this account\n")

+ }

}

// parseSeed splits --seed's raw value on whitespace (strings.Fields) into the

@@ -299,6 +325,9 @@ func runInstancesCreate(args []string) {

}

fmt.Fprintf(stdout, "%s %s (channel %s%s)\n", colorFor(stdout).ok("created instance"),

rec.Host, rec.Channel, environmentSuffix(rec.Environment))

+ if rec.PrivateStoreBucket != "" {

+ mutedf(stdout, " private storage: bucket %s\n", rec.PrivateStoreBucket)

+ }

mutedf(stdout, " the host comes live on the next reconcile pass on its box")

}

diff --git a/instances_privatestorage_test.go b/instances_privatestorage_test.go

new file mode 100644

index 000000000..1531196fe

--- /dev/null

+++ b/instances_privatestorage_test.go

@@ -0,0 +1,83 @@

+package main

+import (

+ "context"

+ "encoding/json"

+ "io"

+ "net/http"

+ "net/http/httptest"

+ "strings"

+ "testing"

+// The three requests `--private-storage` has to keep apart on the wire

+// (issue #412). Absent must be BYTE-IDENTICAL to what an enable sent before

+// the flag existed: a `carlos instances enable -channel beta` that quietly

+// unset a tenant's storage isolation is the failure this pins.

+func TestInstancesEnableSendsPrivateStorageOnlyWhenGiven(t *testing.T) {

+ var gotBody string

+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ b, _ := io.ReadAll(r.Body)

+ gotBody = string(b)

+ json.NewEncoder(w).Encode(map[string]any{

+ "app": "jam", "domain": "jam.example.com", "private_storage": true,

+ })

+ }))

+ defer ts.Close()

+ s := newTestSession(ts.URL)

+ yes, no := true, false

+ m, err := s.instancesEnable(context.Background(), apiAcct{Seg: "bah"}, "jam", nil, nil, &yes)

+ if err != nil {

+ t.Fatalf("instancesEnable: %v", err)

+ }

+ if !strings.Contains(gotBody, `"private_storage":true`) {

+ t.Errorf("request body = %s, want it to ask for private storage", gotBody)

+ }

+ // The ECHO is what the CLI prints, not the flag that was typed: an

+ // enable that never mentioned storage still has to report what the app

+ // does, because that is the fact the next create acts on.

+ if !m.PrivateStorage {

+ t.Error("decoded manifest lost the policy the door echoed")

+ }

+ gotBody = ""

+ if _, err := s.instancesEnable(context.Background(), apiAcct{Seg: "bah"}, "jam", nil, nil, &no); err != nil {

+ t.Fatalf("instancesEnable (off): %v", err)

+ }

+ if !strings.Contains(gotBody, `"private_storage":false`) {

+ t.Errorf("request body = %s, want an explicit false", gotBody)

+ }

+ gotBody = ""

+ if _, err := s.instancesEnable(context.Background(), apiAcct{Seg: "bah"}, "jam", nil, nil, nil); err != nil {

+ t.Fatalf("instancesEnable (absent): %v", err)

+ }

+ if strings.Contains(gotBody, "private_storage") {

+ t.Errorf("request body = %s, want no key at all when the flag was not given", gotBody)

+ }

+// The create response's bucket is decoded, because the line it prints is the

+// only place a member ever learns the name of the bucket their signup path

+// just minted.

+func TestInstanceCreateDecodesThePrivateStoreBucket(t *testing.T) {

+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ w.WriteHeader(http.StatusCreated)

+ json.NewEncoder(w).Encode(map[string]any{

+ "host": "acme.jam.example.com", "channel": "stable",

+ "environment": "acme", "private_store_bucket": "carlos-bah-jam-acme",

+ })

+ }))

+ defer ts.Close()

+ s := newTestSession(ts.URL)

+ rec, err := s.instanceCreate(context.Background(), apiAcct{Seg: "bah"}, "jam",

+ "acme.jam.example.com", "", "", "", "", nil, nil)

+ if err != nil {

+ t.Fatalf("instanceCreate: %v", err)

+ }

+ if rec.PrivateStoreBucket != "carlos-bah-jam-acme" {

+ t.Errorf("decoded bucket = %q, want carlos-bah-jam-acme", rec.PrivateStoreBucket)

+ }