amadan
Merge pull request #413 from carlosframework/412-storage-isolation-config
App policy: every instance gets its own environment and store (#412)
15 files changed,
+1508
−21
apimode.go+19 −4canary.go+1 −1cmd/console/main.go+18 −0deploy.go+1 −1docs/site/cli.md+51 −2docs/superpowers/specs/2026-09-22-instance-private-storage-design.md+276 −0instances_cli.go+37 −8instances_privatestorage_test.go+83 −0internal/console/apiinstances.go+126 −5internal/console/apiinstancesprivatestore_test.go+495 −0internal/console/apistore.go+11 −0internal/console/instanceprivatestore.go+312 −0internal/console/server.go+11 −0internal/provision/provision.go+20 −0internal/provision/provision_test.go+47 −0
CI passed — run details
diff --git a/apimode.go b/apimode.go| index 737cbe2e7..a1d9574d8 100644 |
| --- a/apimode.go |
| +++ b/apimode.go |
| @@ -2769,6 +2769,12 @@ type apiInstanceManifest struct { |
| Channel string `json:"channel"` |
| Regions []string `json:"regions"` |
| Seed []string `json:"seed"` |
| + // PrivateStorage is the app's standing policy that every instance it |
| + // creates gets a config environment of its own and an object store |
| + // bound to it (issue #412). Echoed by the door so `carlos instances |
| + // enable` can print the policy in force rather than the one just |
| + // asked for. |
| + PrivateStorage bool `json:"private_storage"` |
| } |
| // enableError carries the instances-manifest door's STATUS alongside its |
| @@ -2811,11 +2817,12 @@ func enableUnavailable(err error) bool { |
| // pointers for the same reason apiConfigApply's ExecDelivery is: absent |
| // means "preserve what the manifest carries", and seed's explicit empty |
| // slice means "clear" — three requests the wire must keep apart. |
| -func (s *apiSession) instancesEnable(ctx context.Context, a apiAcct, app string, channel *string, seed *[]string) (apiInstanceManifest, error) { |
| +func (s *apiSession) instancesEnable(ctx context.Context, a apiAcct, app string, channel *string, seed *[]string, privateStorage *bool) (apiInstanceManifest, error) { |
| in := struct { |
| - Channel *string `json:"channel,omitempty"` |
| - Seed *[]string `json:"seed,omitempty"` |
| - }{channel, seed} |
| + Channel *string `json:"channel,omitempty"` |
| + Seed *[]string `json:"seed,omitempty"` |
| + PrivateStorage *bool `json:"private_storage,omitempty"` |
| + }{channel, seed, privateStorage} |
| status, body, err := s.putJSON(ctx, apiPath(a, app, "/instances-manifest"), in) |
| if err != nil { |
| return apiInstanceManifest{}, err |
| @@ -2898,6 +2905,14 @@ type apiInstanceRecord struct { |
| // measured it (instanceListRecord.DBBytes). nil when unknown, or from a |
| // console that predates the field — the DB column prints a dash. |
| DBBytes *int64 `json:"db_bytes,omitempty"` |
| + // PrivateStoreBucket is the bucket the CREATE door provisioned for this |
| + // instance under the app's private-storage policy (issue #412). Set on |
| + // exactly one response — the create that minted it — and empty |
| + // everywhere else, including on a list, because it is a fact about what |
| + // one request did rather than a column of the record. What names the |
| + // store afterwards is Environment, and `carlos store status` is what |
| + // reads it. |
| + PrivateStoreBucket string `json:"private_store_bucket"` |
| } |
| // apiUpstreamState mirrors status.UpstreamState's wire shape (internal/status) |
diff --git a/canary.go b/canary.go| index 30c1309ce..da7f63e9c 100644 |
| --- a/canary.go |
| +++ b/canary.go |
| @@ -378,7 +378,7 @@ func runCanaryDeploy(args []string) { |
| // instancesEnable is idempotent and is the ONLY source of the domain |
| // suffix — nothing here composes an apps domain by hand. |
| - m, err := sess.instancesEnable(ctx, acct, app2, nil, nil) |
| + m, err := sess.instancesEnable(ctx, acct, app2, nil, nil, nil) |
| if err != nil { |
| log.Fatalf("canary: enable instances for %s: %v", app2, err) |
| } |
diff --git a/cmd/console/main.go b/cmd/console/main.go| index f8716ab43..ae91ed32e 100644 |
| --- a/cmd/console/main.go |
| +++ b/cmd/console/main.go |
| @@ -293,6 +293,7 @@ func serve() { |
| ScanInterval: envDuration("CARLOS_STORE_SCAN_INTERVAL", 24*time.Hour), |
| ScanMaxSigAge: envDuration("CARLOS_CLAMD_MAX_SIG_AGE", 168*time.Hour), |
| StoreDeployment: storeDeployment, |
| + StoreMaxPerApp: envCount("CARLOS_STORE_MAX_PER_APP", 0), |
| StoreOperators: storeOperators(), |
| // CARLOS_ACME_DELEGATION_ZONE — the same variable the edge reads |
| // (internal/edge/wildcard.go's newWildcardFromEnv) for the same |
| @@ -1005,6 +1006,23 @@ func envDuration(k string, def time.Duration) time.Duration { |
| return d |
| } |
| +// envCount reads a positive count. Zero from an UNSET variable means "the |
| +// console's own default" (the caller supplies def = 0 and the server side |
| +// substitutes); an explicitly set zero or negative is fatal, because a |
| +// deployment that meant to cap something and typed it wrong should be told |
| +// rather than quietly given the default it was trying to change. |
| +func envCount(k string, def int) int { |
| + v := strings.TrimSpace(os.Getenv(k)) |
| + if v == "" { |
| + return def |
| + } |
| + n, err := strconv.Atoi(v) |
| + if err != nil || n <= 0 { |
| + log.Fatalf("console: %s %q must be a positive whole number", k, v) |
| + } |
| + return n |
| +} |
| + |
| // envBytes reads a plain byte count. Same fatal-on-garbage posture. |
| func envBytes(k string, def int64) int64 { |
| v := strings.TrimSpace(os.Getenv(k)) |
diff --git a/deploy.go b/deploy.go| index c13a0ace3..67079baec 100644 |
| --- a/deploy.go |
| +++ b/deploy.go |
| @@ -848,7 +848,7 @@ var errNoStaticHost = errors.New("no static host") |
| // about to promote to, and a record minted on a different one would serve |
| // something else. |
| func mintPlatformHost(ctx context.Context, sess *apiSession, acct apiAcct, app, channel string, messages ...string) (string, error) { |
| - m, err := sess.instancesEnable(ctx, acct, app, nil, nil) |
| + m, err := sess.instancesEnable(ctx, acct, app, nil, nil, nil) |
| if err != nil { |
| return "", err |
| } |
diff --git a/docs/site/cli.md b/docs/site/cli.md| index 3df903a7b..6abbbc90f 100644 |
| --- a/docs/site/cli.md |
| +++ b/docs/site/cli.md |
| @@ -580,6 +580,53 @@ bind it here, then restart the instance to pick the values up. |
| carlos instances set-environment --app jam --host new-queue.jam.bab.oncarlos.com --environment jam-canary |
| ``` |
| +#### Storage of its own, for every instance |
| + |
| +`--private-storage` on `enable` is one declaration that changes what every |
| +later `create` does. The app says, once, that each of its instances gets a |
| +config environment of its own with an object store bound to it — and from |
| +then on `carlos instances create` derives the environment from the host's |
| +first label, provisions that environment's bucket and IAM user, and delivers |
| +the coordinates into the bundle the new instance reads. No `store create`, no |
| +operator, no second command. |
| + |
| +```sh |
| +carlos instances enable --app jam --private-storage |
| +carlos instances create --app jam --host acme.jam.bab.oncarlos.com |
| +# created instance acme.jam.bab.oncarlos.com (channel stable, environment acme) |
| +# private storage: bucket carlos-bab-jam-acme |
| +``` |
| + |
| +This is the shape for a managed product with one instance per customer, and |
| +the split is the point: declaring the policy is a signed-in person's act that |
| +no service credential can perform, while spending it is the ordinary create |
| +door an `instance`-role credential already reaches. So a public signup path |
| +provisions a tenant's bucket without ever holding the power to create one. |
| + |
| +Three things to know before you turn it on: |
| + |
| +- The account needs an **approved storage entitlement** (`carlos store |
| + apply`) and the deployment needs object storage configured. `enable` |
| + refuses the policy rather than letting it fail once per signup. |
| +- The host's first label becomes the environment name, so it must be a legal |
| + one: lowercase letters, digits and hyphens, at most 32 characters, and not |
| + `default`. A label that cannot name a store is refused at `create`, before |
| + anything is written. |
| +- **Nothing is ever torn down automatically.** Deleting an instance leaves |
| + its bucket, exactly as it already leaves the parked database. `carlos store |
| + status --app <a>` lists every store the app has, which is where an |
| + abandoned one is visible; destroying it is the console's own action, behind |
| + the typed bucket name and a passkey. |
| + |
| +An app may have at most 1000 environment stores by default; a deployment sets |
| +its own limit with `CARLOS_STORE_MAX_PER_APP`. The limit exists because each |
| +store is an IAM user as well as a bucket, and AWS allows 5,000 IAM users per |
| +account — so it bounds one app, not the deployment. Five apps at the default |
| +would reach that quota between them. |
| + |
| +`--private-storage=false` turns the policy off. Instances created before it |
| +keep the bindings and buckets they already have. |
| + |
| Typed on a box with no sub-verb, `carlos instances` does something different: |
| it lists that box's own registry routes, backing and owning unit included. |
| Off the box, [carlos routes](/docs/cli#carlos-routes) answers the same |
| @@ -1262,8 +1309,10 @@ one app, which also means it cannot create apps. |
| One door stays shut to every role: `carlos instances enable`, which writes the |
| manifest bounding which domain, pool and regions an instance may be created |
| -in. A person enables the app once and the credential provisions inside those |
| -bounds afterwards. |
| +in — and, with `--private-storage`, whether each instance gets an object store |
| +of its own. A person enables the app once and the credential provisions inside |
| +those bounds afterwards, including minting a tenant's bucket it could never |
| +have created directly. |
| The secret prints once, same as a fleet token, and the same distinction |
| applies at the other end: `rotate` refuses the credential's next call with |
diff --git a/docs/superpowers/specs/2026-09-22-instance-private-storage-design.md b/docs/superpowers/specs/2026-09-22-instance-private-storage-design.md| new file mode 100644 |
| index 000000000..518e979cd |
| --- /dev/null |
| +++ b/docs/superpowers/specs/2026-09-22-instance-private-storage-design.md |
| @@ -0,0 +1,276 @@ |
| +# Private storage per instance: policy on the app, provisioning by the platform |
| + |
| +**Date:** 2026-09-22 |
| +**Status:** implemented in this branch. Scope confirmed with James on |
| +2026-09-22: §5 (the policy), §6 (the derived name), §7 (provisioning at |
| +create) and §8 (the ceiling). §10's two items are NOT built and are named |
| +there so they are not mistaken for shipped. |
| +**Issue:** GitHub #412 (a better answer to #410, which this makes |
| +unnecessary). |
| +**Builds on:** `2026-09-19-per-environment-store-design.md` (#408) — the |
| +per-environment store primitive this spends. Nothing here works without it. |
| + |
| +## 1. The problem, as found |
| + |
| +#408 gave a store an environment to belong to. It did not give anyone a way |
| +to reach one without a person. |
| + |
| +Creating an environment store is three acts: `carlos store create --app a |
| +--environment e`, then a provisioning walk, then binding an instance record |
| +to `e`. The first two run behind `apiAccountApp` plus the account |
| +entitlement, and the CLI verb that drives them is a member's. An |
| +`instance`-role service credential — the one a public signup path can hold — |
| +can create an instance record and nothing else. |
| + |
| +So every signup on Managed Jelly (lazyatom/jam#187) stops for a person to |
| +run one command. #410 asked for the fix that widens the role. This is the |
| +fix that widens nobody: **the human declares the policy once, on the app, |
| +and the create door does the rest under that standing declaration.** |
| + |
| +### What is already right, and must stay right |
| + |
| +- **The entitlement is per ACCOUNT** (`store/entitlements/<sqid>.json`), and |
| + #408 §6 already says that in an approved account, creating an environment |
| + store provisions it with no new approval. That sentence is the whole |
| + authority story here; this spec adds no new one. |
| +- **A store is keyed on `(account, app, environment)`** (#408 §3, ruling). |
| + Deriving the environment from the host at create time keeps that key |
| + exactly and adds no per-host config source. |
| +- **Teardown is explicit** (#408 §3, ruling). Nothing here destroys |
| + anything, and §10 says what that costs. |
| +- **The create door's refusal order** — gate, budget, manifest, host, |
| + region, channel, write — and its byte-identical 404 before membership is |
| + proven. Everything this spec adds sits AFTER the manifest read, so a |
| + non-member still cannot learn that an app carries the policy. |
| + |
| +## 2. The shape, in one paragraph |
| + |
| +An app may declare that **every instance it creates gets a config |
| +environment of its own, with an object store bound to that environment**. |
| +The declaration is a bool on the instances manifest, written by |
| +`carlos instances enable --app a --private-storage` — a signed-in person's |
| +act, which no service credential can perform. From then on `POST |
| +…/instances` derives the environment from the host's first label, provisions |
| +that environment's store under the account's existing entitlement, and binds |
| +the record to it. An `instance`-role credential gains nothing it could not |
| +already do; it simply lands in a world where the app's policy has already |
| +been decided. |
| + |
| +## 3. Rulings |
| + |
| +- **(ruling, James 2026-09-22) The policy is declared on the app, and only a |
| + person may declare it.** `handleAPIInstanceEnable` is registered with the |
| + zero Set and gated by `apiPerson`; that is the wall, and it is the same |
| + wall that makes this a better answer than #410. |
| +- **(ruling, James 2026-09-22) Scope is the policy, the derived name and the |
| + ceiling.** The residue listing (#412's point 2) and holding an instance's |
| + first wake until its store is delivered (#412's point 4) are follow-up |
| + work, named in §10. |
| + |
| +## 4. Non-goals |
| + |
| +- Automatic teardown of anything (#408 §3's ruling, unchanged). |
| +- A residue listing for deleted instances (#412 point 2). §10. |
| +- Gating an instance's first wake on delivery (#412 point 4). §10. |
| +- Per-store usage attribution, scanning environment stores — #408's own |
| + non-goals, unchanged. |
| +- An account-wide (rather than per-app) store ceiling. §8 says why the cap |
| + is shaped per app, what that leaves unbounded, and what a real one costs. |
| +- Per-instance secrets. The derived environment is the primitive they would |
| + hang off; this spec builds the primitive and stops. |
| + |
| +## 5. The policy |
| + |
| +**(decision) One additive bool on `provision.Manifest`.** |
| + |
| +```go |
| +// PrivateStorage is the app's standing policy that every instance created |
| +// under this manifest gets a config environment of its own and an object |
| +// store bound to it. Absent/false is what every manifest in a production |
| +// bucket says today, and means exactly today's behaviour. |
| +PrivateStorage bool `json:"private_storage,omitempty"` |
| +``` |
| + |
| +A plain bool, not a pointer, and the difference from `Record.Hibernate` and |
| +`Record.StatusNamed` is worth stating because those two are pointers for a |
| +reason that does not apply here. Those fields are CONVERGED — a box reads |
| +them every pass, so "absent" has to be distinguishable from "an explicit |
| +false" or the first pass after deploy silently acts on a value nobody wrote. |
| +This field is read at one door, at create time, and false IS the legacy |
| +behaviour byte for byte. There is nothing for absence to mean that false |
| +does not already mean. |
| + |
| +**(decision) The enable door takes it as a POINTER on the wire** — |
| +`private_storage` absent preserves what the manifest carries, present |
| +replaces it — matching `channel` and `seed`, which are pointers on that |
| +door for exactly this reason. `carlos instances enable --private-storage` |
| +sends true; `--private-storage=false` sends false; omitting the flag sends |
| +nothing. |
| + |
| +**(decision) The door refuses a policy it knows cannot be honoured.** Two |
| +checks, both before the manifest is written: |
| + |
| +- The deployment has no object-storage backend (`CARLOS_STORE_BACKEND` |
| + unset) → 503 naming the variable. |
| +- The account's entitlement is not approved → 403 naming `carlos store |
| + apply`. |
| + |
| +This is a departure from the enable door's habit of writing whatever policy |
| +it is handed, and it is deliberate. The policy's entire purpose is that |
| +creates afterwards succeed with no person in the loop. A manifest that |
| +guarantees every future create fails is not policy, it is a trap that fires |
| +once per signup, and the two sentences above are the only two that name the |
| +fix. Turning the policy OFF is never refused — an account whose entitlement |
| +was later declined must be able to get back to a working create door. |
| + |
| +## 6. The derived name |
| + |
| +**(decision) The environment is the host's first label** — `Record.Slug`, |
| +which the create door already computes as `strings.TrimSuffix(host, "." + |
| +m.Domain)`. It is the one name that is predictable from the host, already |
| +unique within the app (the record key is the host), and readable in `store |
| +status` and the console's Environments view at a hundred of them, which is |
| +#412's point 1 in full. |
| + |
| +**(decision) An explicit `environment` in the request body wins.** The |
| +policy fills a blank; it does not seize the axis. A caller who names an |
| +environment gets a store for THAT environment — which is still the |
| +invariant this spec exists for, since what is promised is that the record's |
| +bundle carries `CARLOS_STORE_*`, not that the bundle has any particular |
| +name. It keeps `carlos canary` working on a private-storage app, and two |
| +instances naming one environment share its store exactly as they share its |
| +config, which #408 §5 already rules is visible rather than refused. |
| + |
| +**(decision) Three names are checked BEFORE anything is written**, because |
| +a DNS label is not an environment name and an environment name is not |
| +always spendable: |
| + |
| +| check | why it can fail | |
| +| --- | --- | |
| +| `appconfig.ValidEnvironmentName(slug)` | a label may be 63 characters, an environment name 32; and `default` is a legal label but names the app's own bundle | |
| +| `objstore.ValidBucketName(MintedBucketIn(...))` | S3's 63 | |
| +| `objstore.ValidUserName(UserNameIn(...))` | IAM's 64 | |
| + |
| +Each refuses with 400 and a sentence naming the part the caller can |
| +shorten. The walk checks the last two itself (#408's implementation gate), |
| +but reaching them there would mean an instance record already written over |
| +an estate that cannot exist. |
| + |
| +## 7. Provisioning, and the order it happens in |
| + |
| +**(ruling, James 2026-09-22 — implied by the scope choice, stated here |
| +because it is the load-bearing decision of this file) The store is |
| +provisioned BEFORE the record is written, and a provisioning failure means |
| +no record at all.** |
| + |
| +The two orders fail differently, and only one of them fails visibly: |
| + |
| +- *Record first.* Provisioning then fails, and an instance exists whose |
| + bundle has no `CARLOS_STORE_*`. It wakes, finds no coordinates, and runs |
| + on local disk — which is the exact outcome the policy was declared to |
| + prevent, arrived at silently, with no verb to repair it (there is no |
| + `carlos instances create --resume`). |
| +- *Store first.* The record write then fails, and a declared-or-provisioned |
| + store exists for an environment no instance is bound to. It is on the |
| + screen in `store status`, and — because the environment is derived from |
| + the host — **a retry of the same host reuses it exactly**: `GrantIn` is |
| + idempotent, so the second attempt short-circuits and costs nothing. |
| + |
| +So the property this spec asserts is: **no instance record is ever created |
| +under this policy without its store already provisioned and delivered.** |
| +That is stated as such because it is the whole feature; a create that |
| +half-succeeded would be worse than one that refused. |
| + |
| +One window stays open and is named rather than papered over. The create door |
| +already writes the record and then re-reads the sibling-apex check, deleting |
| +the record if a sibling app claimed the label mid-create. On that path the |
| +store survives the record. It is logged in those words, beside the existing |
| +`UNDO FAILED` line, because the alternative — tearing down a bucket from a |
| +code path nobody drove — would make this the platform's first automatic |
| +destroyer of tenant data, which #408 §3 rules out. |
| + |
| +**(decision) The walk runs exactly as `handleAPIStoreProvision` runs it**: |
| +the per-store walk lock (`storeWalks.lock(sqid, app, env)`), the seal key, |
| +`storeGrantTimeout`, `GrantSourceAccountApproval`, and the entitlement |
| +re-read after the lock. Not a second implementation of provisioning — the |
| +same calls in the same order, factored into one helper both doors use, so |
| +the two cannot drift. |
| + |
| +**(decision) The env prefix is inherited from the app's own store.** A |
| +private-storage app that already declared `carlos store create --env-prefix |
| +JELLY_S3` reads those names in its code; an instance store delivering |
| +`CARLOS_STORE_BUCKET` into the same app would be a second vocabulary for one |
| +thing. With no app store declared, the default `CARLOS_STORE` prefix is |
| +taken, which is what a fresh declaration would take. |
| + |
| +## 8. The ceiling |
| + |
| +**(decision) A cap on how many ENVIRONMENT stores one app may have**, |
| +`CARLOS_STORE_MAX_PER_APP`, default 1000. A bucket plus an IAM user per |
| +instance is a bigger footprint than an instance, and the binding limit is |
| +IAM's: 5,000 users per cloud account, an adjustable quota |
| +([AWS service quotas](https://docs.aws.amazon.com/general/latest/gr/iam-service.html)). |
| +A signup path that can mint them needs a bar it stops at. |
| + |
| +**(correction, James 2026-09-23) The first draft of this section said 100, |
| +and that was a number picked for tidiness rather than derived from the |
| +quota.** A fifth of one app's fair share would have refused an ordinary |
| +managed product at its hundredth customer — a real refusal in exchange for |
| +no real protection. 1000 is a fifth of the account quota, so one app can run |
| +away without exhausting it while five cannot. |
| + |
| +**What the cap does not bound**, stated because the per-app shape invites the |
| +wrong reading: five apps at the default reach 5,000 between them, and the |
| +deployment's own IAM principals are not in that sum. This is a brake on one |
| +runaway signup path, not a budget for the cloud account. An account-wide |
| +bound needs a counter kept on the side rather than one derived by listing, |
| +and §4 lists it as a non-goal. |
| + |
| +Per app, not per account, for a reason of cost: the count comes from one |
| +`objstore.StoreEnvironments` prefix listing of the app, which the create |
| +door can afford. A deployment-wide count would mean listing every app's |
| +stores on every create. |
| + |
| +Two properties: |
| + |
| +- **It is enforced on both doors**, `POST …/instances` and `POST |
| + …/store?environment=`. A ceiling only the automatic path respected would |
| + be a ceiling anyone could step over by hand. |
| +- **Re-using an environment that already has a store is never capped.** The |
| + cap is on the FOOTPRINT, and a second instance bound to an existing |
| + environment adds no bucket. Without this, a retry of a create that failed |
| + after provisioning would be refused by the cap it had just reached. |
| + |
| +The refusal is a 409 naming the count and the cap. |
| + |
| +## 9. What a member sees |
| + |
| +``` |
| +carlos instances enable --app a --private-storage |
| +carlos instances create --app a --host acme.a.<sqid>.<apps-domain> |
| + created instance acme.a.… (channel stable, environment acme) |
| + private storage: bucket carlos-bab-a-acme |
| +``` |
| + |
| +and `carlos store status --app a` lists the acme store beside every other, |
| +which is #408 §5's listing doing its job unchanged. |
| + |
| +## 10. Not built, and named so it is not mistaken for shipped |
| + |
| +- **#412's point 2, the residue listing.** `instances delete` leaves the |
| + parked database and blobs today, and after this change it also leaves an |
| + environment bundle and a store. The issue says this "probably should not |
| + ship without at least a way to list what deleted instances left behind", |
| + and that judgement stands: the residue is now created automatically, so it |
| + accrues once per cancelled customer. What exists in the meantime is `store |
| + status`, which lists every store of the app including ones no instance |
| + reads — the honest half of #408 §8, and enough to find an orphan if you go |
| + looking. The follow-up is a verb that does the looking. |
| +- **#412's point 4, holding the wake.** The provisioning walk completes |
| + before the record is written (§7), so the coordinates are in the bundle |
| + before the record a box could reconcile from exists at all. That is |
| + strictly better than the manager's "refuse the seed until the bucket is |
| + seen" workaround, and it is not the same as a guarantee: delivery is a |
| + config-bundle write and the box's sync pass is what carries it to the |
| + instance, so a wake racing the first sync can still find no coordinates. |
| + Closing that needs the box side, and it is not in this branch. |
diff --git a/instances_cli.go b/instances_cli.go| index 5ffc16865..fe631ed67 100644 |
| --- a/instances_cli.go |
| +++ b/instances_cli.go |
| @@ -59,6 +59,11 @@ func runInstancesEnable(args []string) { |
| channel := channelFlag(fs, "release channel stamped into every record at create (store-mode default stable; signed-in mode: omitted preserves the manifest's)") |
| regions := fs.String("regions", "", "comma-separated region allow-list (store mode only); empty = any shape-valid region") |
| seed := fs.String("seed", "", "argv of the app's first-boot init command, whitespace-split (strings.Fields)") |
| + // A bool flag rather than a string, read through flagWasSet for the |
| + // same absent-vs-explicit split --channel and --seed use: omitting it |
| + // preserves whatever policy the manifest already carries, and |
| + // --private-storage=false turns the policy off. |
| + privateStorage := fs.Bool("private-storage", false, "give every instance of this app a config environment and an object store of its own (issue #412); --private-storage=false turns the policy off") |
| consoleFlag := consoleOption(fs, "console to enable through (signed-in mode only; default: the only logged-in one)") |
| acctRaw, acctIDRaw, acctOf := accountFlags(fs) |
| fs.Usage = flagUsage(fs, instancesEnableUsage) |
| @@ -83,6 +88,10 @@ func runInstancesEnable(args []string) { |
| if flagWasSet(fs, "channel") { |
| channelReq = channel |
| } |
| + var privateStorageReq *bool |
| + if flagWasSet(fs, "private-storage") { |
| + privateStorageReq = privateStorage |
| + } |
| var seedReq *[]string |
| if flagWasSet(fs, "seed") { |
| // parseSeed("") is an empty (non-nil after this) argv: an explicit |
| @@ -97,11 +106,17 @@ func runInstancesEnable(args []string) { |
| printUsing(stdout, sess) |
| acct := resolvedAccount(sess, acctRaw, acctIDRaw) |
| app2 := resolvedApp(sess, acct, *app, instancesEnableUsage) |
| - m, err := sess.instancesEnable(context.Background(), acct, app2, channelReq, seedReq) |
| + m, err := sess.instancesEnable(context.Background(), acct, app2, channelReq, seedReq, privateStorageReq) |
| if err != nil { |
| log.Fatalf("instances enable: %v", err) |
| } |
| fmt.Fprintf(stdout, "%s for %s under %s\n", colorFor(stdout).ok("enabled instances"), m.App, m.Domain) |
| + // The policy IN FORCE, from the door's echo, not the flag that was |
| + // typed: an enable that never mentioned storage still prints what |
| + // the app does, which is the fact the next create will act on. |
| + if m.PrivateStorage { |
| + mutedf(stdout, " every instance gets its own config environment and object store\n") |
| + } |
| mutedf(stdout, " first instance: carlos instances create --app %s --host <label>.%s\n", m.App, m.Domain) |
| return |
| } |
| @@ -148,18 +163,29 @@ func runInstancesEnable(args []string) { |
| } |
| } |
| m := provision.Manifest{ |
| - Schema: 1, |
| - App: *app, |
| - Domain: *domain, |
| - Pool: *pool, |
| - Channel: *channel, |
| - Regions: regionList, |
| - Seed: parseSeed(*seed), |
| + Schema: 1, |
| + App: *app, |
| + Domain: *domain, |
| + Pool: *pool, |
| + Channel: *channel, |
| + Regions: regionList, |
| + Seed: parseSeed(*seed), |
| + PrivateStorage: *privateStorage, |
| } |
| if err := instancesEnable(context.Background(), openStore(), acctOf(), *app, m); err != nil { |
| log.Fatal(err) |
| } |
| fmt.Printf("%s for %s under %s\n", colorFor(os.Stdout).ok("enabled instances"), *app, *domain) |
| + if *privateStorage { |
| + // Store mode writes the manifest straight to the bucket, so nothing |
| + // checked that this deployment HAS object storage or that the |
| + // account is approved for it — the console's enable door does both |
| + // and refuses, but the operator path has no console to ask. Named |
| + // here so the policy is not discovered to be inert at the first |
| + // signup. |
| + mutedf(os.Stdout, " every instance gets its own config environment and object store\n") |
| + mutedf(os.Stdout, " the console minting them needs CARLOS_STORE_BACKEND and an approved store entitlement for this account\n") |
| + } |
| } |
| // parseSeed splits --seed's raw value on whitespace (strings.Fields) into the |
| @@ -299,6 +325,9 @@ func runInstancesCreate(args []string) { |
| } |
| fmt.Fprintf(stdout, "%s %s (channel %s%s)\n", colorFor(stdout).ok("created instance"), |
| rec.Host, rec.Channel, environmentSuffix(rec.Environment)) |
| + if rec.PrivateStoreBucket != "" { |
| + mutedf(stdout, " private storage: bucket %s\n", rec.PrivateStoreBucket) |
| + } |
| mutedf(stdout, " the host comes live on the next reconcile pass on its box") |
| } |
diff --git a/instances_privatestorage_test.go b/instances_privatestorage_test.go| new file mode 100644 |
| index 000000000..1531196fe |
| --- /dev/null |
| +++ b/instances_privatestorage_test.go |
| @@ -0,0 +1,83 @@ |
| +package main |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "io" |
| + "net/http" |
| + "net/http/httptest" |
| + "strings" |
| + "testing" |
| +) |
| + |
| +// The three requests `--private-storage` has to keep apart on the wire |
| +// (issue #412). Absent must be BYTE-IDENTICAL to what an enable sent before |
| +// the flag existed: a `carlos instances enable -channel beta` that quietly |
| +// unset a tenant's storage isolation is the failure this pins. |
| +func TestInstancesEnableSendsPrivateStorageOnlyWhenGiven(t *testing.T) { |
| + var gotBody string |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + b, _ := io.ReadAll(r.Body) |
| + gotBody = string(b) |
| + json.NewEncoder(w).Encode(map[string]any{ |
| + "app": "jam", "domain": "jam.example.com", "private_storage": true, |
| + }) |
| + })) |
| + defer ts.Close() |
| + s := newTestSession(ts.URL) |
| + yes, no := true, false |
| + |
| + m, err := s.instancesEnable(context.Background(), apiAcct{Seg: "bah"}, "jam", nil, nil, &yes) |
| + if err != nil { |
| + t.Fatalf("instancesEnable: %v", err) |
| + } |
| + if !strings.Contains(gotBody, `"private_storage":true`) { |
| + t.Errorf("request body = %s, want it to ask for private storage", gotBody) |
| + } |
| + // The ECHO is what the CLI prints, not the flag that was typed: an |
| + // enable that never mentioned storage still has to report what the app |
| + // does, because that is the fact the next create acts on. |
| + if !m.PrivateStorage { |
| + t.Error("decoded manifest lost the policy the door echoed") |
| + } |
| + |
| + gotBody = "" |
| + if _, err := s.instancesEnable(context.Background(), apiAcct{Seg: "bah"}, "jam", nil, nil, &no); err != nil { |
| + t.Fatalf("instancesEnable (off): %v", err) |
| + } |
| + if !strings.Contains(gotBody, `"private_storage":false`) { |
| + t.Errorf("request body = %s, want an explicit false", gotBody) |
| + } |
| + |
| + gotBody = "" |
| + if _, err := s.instancesEnable(context.Background(), apiAcct{Seg: "bah"}, "jam", nil, nil, nil); err != nil { |
| + t.Fatalf("instancesEnable (absent): %v", err) |
| + } |
| + if strings.Contains(gotBody, "private_storage") { |
| + t.Errorf("request body = %s, want no key at all when the flag was not given", gotBody) |
| + } |
| +} |
| + |
| +// The create response's bucket is decoded, because the line it prints is the |
| +// only place a member ever learns the name of the bucket their signup path |
| +// just minted. |
| +func TestInstanceCreateDecodesThePrivateStoreBucket(t *testing.T) { |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + w.WriteHeader(http.StatusCreated) |
| + json.NewEncoder(w).Encode(map[string]any{ |
| + "host": "acme.jam.example.com", "channel": "stable", |
| + "environment": "acme", "private_store_bucket": "carlos-bah-jam-acme", |
| + }) |
| + })) |
| + defer ts.Close() |
| + s := newTestSession(ts.URL) |
| + |
| + rec, err := s.instanceCreate(context.Background(), apiAcct{Seg: "bah"}, "jam", |
| + "acme.jam.example.com", "", "", "", "", nil, nil) |
| + if err != nil { |
| + t.Fatalf("instanceCreate: %v", err) |
| + } |
| + if rec.PrivateStoreBucket != "carlos-bah-jam-acme" { |
| + t.Errorf("decoded bucket = %q, want carlos-bah-jam-acme", rec.PrivateStoreBucket) |
| + } |
| +} |
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go| index 1256c0024..48e87f5d4 100644 |
| --- a/internal/console/apiinstances.go |
| +++ b/internal/console/apiinstances.go |
| @@ -536,6 +536,12 @@ type apiRecordView struct { |
| provision.Record |
| HiddenGate string `json:"gate,omitempty"` |
| Gated bool `json:"gated"` |
| + // PrivateStoreBucket is the bucket the create door provisioned for this |
| + // instance under the app's private-storage policy (issue #412), empty on |
| + // every other record and every other door. Not part of the record: it is |
| + // a fact about what this one request did, and the record's own |
| + // Environment is what names the store afterwards. |
| + PrivateStoreBucket string `json:"private_store_bucket,omitempty"` |
| } |
| func recordView(rec provision.Record) apiRecordView { |
| @@ -826,6 +832,33 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| if rec.Kind == "static" && platformNamespace && rec.Pool == "" { |
| rec.Steering = "latency" |
| } |
| + // The app's private-storage policy (issue #412, spec §6): every instance |
| + // gets a config environment of its own, named for the host's first |
| + // label, with an object store bound to it. |
| + // |
| + // An EXPLICIT environment in the request wins, and the policy still |
| + // applies to it. The policy fills a blank; it does not seize the axis. |
| + // What it promises is that this record's bundle carries the store |
| + // coordinates — not that the bundle has any particular name — so |
| + // honouring the caller's own name keeps `carlos canary` working on such |
| + // an app, and two instances naming one environment share its store |
| + // exactly as they already share its config (#408 §5: sharing is |
| + // visible, not refused). |
| + // |
| + // Stamped BEFORE ValidateShape so the shape rail sees the same |
| + // environment the box will, and before the ceiling and the walk below so |
| + // neither can be asked about a record that was never legal. |
| + if m.PrivateStorage { |
| + env, derived := rec.Environment, false |
| + if env == "" { |
| + env, derived = rec.Slug, true |
| + } |
| + if ref := s.instanceStoreNames(acct, app, env, derived); ref != nil { |
| + jsonError(w, ref.Status, ref.Msg) |
| + return |
| + } |
| + rec.Environment = env |
| + } |
| // The shape rails, at the door — kind vocabulary, one upstream shape per |
| // record, addr syntax, and the combinations that would wedge a host (a |
| // static route with an upstream, a hibernating addr-backed one). The |
| @@ -837,6 +870,47 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| jsonError(w, http.StatusBadRequest, err.Error()) |
| return |
| } |
| + // THE STORE IS PROVISIONED BEFORE THE RECORD IS WRITTEN, and a failure |
| + // here means no record at all (issue #412, spec §7). This ordering is |
| + // the load-bearing decision of the feature, so it is argued here rather |
| + // than assumed: |
| + // |
| + // Record first, and a failed walk leaves an instance whose bundle has no |
| + // store coordinates. It wakes, finds none, and runs on local disk — the |
| + // exact outcome the policy was declared to prevent, arrived at silently, |
| + // with no verb to repair it (there is no `instances create --resume`). |
| + // |
| + // Store first, and a failed record write leaves a store for an |
| + // environment no instance is bound to. That is visible — `store status` |
| + // lists every store of the app — and, because the environment is DERIVED |
| + // FROM THE HOST, a retry of the same host reuses it exactly: GrantIn is |
| + // idempotent, so the second attempt short-circuits and costs nothing. |
| + // |
| + // So: no instance record is ever created under this policy without its |
| + // store already provisioned and delivered. |
| + privateBucket := "" |
| + if m.PrivateStorage { |
| + if ref := s.storeCeiling(ctx, acct, app, rec.Environment); ref != nil { |
| + jsonError(w, ref.Status, ref.Msg) |
| + return |
| + } |
| + // r.Context(), not ctx: the walk takes storeGrantTimeout, which is |
| + // longer than this handler's own deadline. |
| + storeRec, ok := s.instanceStoreProvision(r.Context(), w, acct, app, rec.Environment, principal) |
| + if !ok { |
| + return |
| + } |
| + privateBucket = storeBucketName(storeRec) |
| + // And a fresh deadline for what is left. The walk can have consumed |
| + // every second of the handler's original budget, and the record |
| + // write below inheriting a spent context would fail immediately |
| + // after the bucket had already been paid for — the one shape of |
| + // failure this whole ordering exists to avoid. |
| + cancel() |
| + var recancel context.CancelFunc |
| + ctx, recancel = context.WithTimeout(r.Context(), instanceStoreTimeout) |
| + defer recancel() |
| + } |
| if err := provision.CreateRecord(ctx, s.cfg.Releases, rec); err != nil { |
| if errors.Is(err, release.ErrCASConflict) { |
| // A lost race against a second signup for the same host, or a |
| @@ -858,6 +932,15 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| if derr := provision.DeleteRecord(ctx, s.cfg.Releases, acct.Sqid, app, host); derr != nil { |
| log.Printf("console: api instances create %s/%s/%s: %s was claimed mid-create and UNDO FAILED: %v", acct.Sqid, app, host, siblingLabel, derr) |
| } |
| + if privateBucket != "" { |
| + // The one window where this feature's ordering leaves |
| + // residue, logged in those words rather than papered over: |
| + // tearing the bucket down from a path nobody drove would |
| + // make this the platform's first automatic destroyer of |
| + // tenant data, which the per-environment store spec rules |
| + // out (#408 §3). It is listed by `carlos store status`. |
| + log.Printf("console: api instances create %s/%s/%s: %s was claimed mid-create; the record was withdrawn and its store %s WAS LEFT BEHIND", acct.Sqid, app, host, siblingLabel, privateBucket) |
| + } |
| jsonError(w, http.StatusConflict, |
| host+" is app "+siblingLabel+"'s platform apex — it was created while this instance was being written; the instance was not kept") |
| return |
| @@ -877,7 +960,9 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| // a box-consumed write (carlos-deploy §3): announce it, and the instance |
| // is serving seconds after signup instead of on the hour. |
| s.bumpSerial(ctx) |
| - writeJSON(w, http.StatusCreated, recordView(rec)) |
| + out := recordView(rec) |
| + out.PrivateStoreBucket = privateBucket |
| + writeJSON(w, http.StatusCreated, out) |
| } |
| // instanceChannel answers "which channel is this record born on" for a |
| @@ -1472,9 +1557,15 @@ func (s *Server) handleAPIInstanceEnable(w http.ResponseWriter, r *http.Request) |
| // different requests on the wire (the apiConfigApply precedent): an |
| // omitted field preserves what the manifest already carries, an explicit |
| // value replaces it — including seed's explicit [] meaning "clear". |
| + // |
| + // PrivateStorage joined them on 2026-09-22 (issue #412) as a pointer for |
| + // exactly the same reason: an app that already gives its instances their |
| + // own storage must not lose the policy to a `carlos instances enable |
| + // -channel beta` that never mentioned it. |
| var req struct { |
| - Channel *string `json:"channel"` |
| - Seed *[]string `json:"seed"` |
| + Channel *string `json:"channel"` |
| + Seed *[]string `json:"seed"` |
| + PrivateStorage *bool `json:"private_storage"` |
| } |
| if !readJSON(w, r, &req) { |
| return |
| @@ -1547,6 +1638,21 @@ func (s *Server) handleAPIInstanceEnable(w http.ResponseWriter, r *http.Request) |
| if req.Seed != nil { |
| m.Seed = *req.Seed |
| } |
| + // Turning the policy ON is checked against what this deployment and this |
| + // account can actually do; turning it OFF never is (spec §5). The |
| + // asymmetry is deliberate: a policy that cannot be honoured fails once |
| + // per signup in a door whose caller is a machine, so the person |
| + // declaring it is told now — but an account whose entitlement was later |
| + // declined must be able to get its create door working again without |
| + // waiting on an operator. |
| + if req.PrivateStorage != nil && *req.PrivateStorage && !m.PrivateStorage { |
| + if !s.instancePrivateStorageAllowed(ctx, w, acct) { |
| + return |
| + } |
| + } |
| + if req.PrivateStorage != nil { |
| + m.PrivateStorage = *req.PrivateStorage |
| + } |
| if err := provision.PutManifestIfMatch(ctx, s.cfg.Releases, acct.Sqid, app, *m, etag); err != nil { |
| if errors.Is(err, release.ErrCASConflict) { |
| jsonError(w, http.StatusConflict, |
| @@ -1557,16 +1663,31 @@ func (s *Server) handleAPIInstanceEnable(w http.ResponseWriter, r *http.Request) |
| jsonError(w, http.StatusBadGateway, "could not enable instances — try again") |
| return |
| } |
| - s.audit(acct, app, addr, "instance-enable", "enabled instances under "+domain+" (cli)") |
| + s.audit(acct, app, addr, "instance-enable", "enabled instances under "+domain+privateStorageAuditSuffix(req.PrivateStorage)+" (cli)") |
| // The manifest names the channel every minted route follows, so a change |
| // here changes what boxes converge onto — reconcile reads it each pass. |
| s.bumpSerial(ctx) |
| writeJSON(w, http.StatusOK, map[string]any{ |
| "app": app, "domain": m.Domain, "pool": m.Pool, "channel": m.Channel, |
| - "regions": m.Regions, "seed": m.Seed, |
| + "regions": m.Regions, "seed": m.Seed, "private_storage": m.PrivateStorage, |
| }) |
| } |
| +// privateStorageAuditSuffix names the policy in the audit line only when |
| +// this request SPOKE to it. An enable that changed a channel says nothing |
| +// about storage, and a row that restated the standing policy every time |
| +// would make the one request that changed it impossible to find. |
| +func privateStorageAuditSuffix(req *bool) string { |
| + switch { |
| + case req == nil: |
| + return "" |
| + case *req: |
| + return ", every instance with storage of its own" |
| + default: |
| + return ", instances sharing the app's storage" |
| + } |
| +} |
| + |
| // instanceRegionAllowed applies the manifest's region allow-list. An empty |
| // list means any shape-valid region (spec §3.1) — including the empty, |
| // unconstrained one, which is what a single-region deployment uses. |
diff --git a/internal/console/apiinstancesprivatestore_test.go b/internal/console/apiinstancesprivatestore_test.go| new file mode 100644 |
| index 000000000..b11754521 |
| --- /dev/null |
| +++ b/internal/console/apiinstancesprivatestore_test.go |
| @@ -0,0 +1,495 @@ |
| +package console |
| + |
| +// The app-declared private-storage policy (issue #412). What these pin, in |
| +// the order they would hurt if lost: |
| +// |
| +// - the create door provisions the store BEFORE it writes the record, so |
| +// a failed walk leaves no instance at all — never an instance that wakes |
| +// onto local disk because its bundle had no coordinates; |
| +// - the environment is the host's first label, and the coordinates land in |
| +// THAT bundle, not the app's default one; |
| +// - an app without the policy is byte-for-byte unchanged; |
| +// - the ceiling holds on both doors, and re-using an existing store is not |
| +// capped; |
| +// - declaring the policy is a person's act, refused in the two states |
| +// where it could only ever be a trap. |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "net/http" |
| + "os" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/appconfig" |
| + "github.com/carlosframework/platform/internal/objstore" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// privateStorageFixture is instanceFixture with the store plane armed: a |
| +// backend, a sealing key, and an approved account entitlement — the three |
| +// things the policy spends, and the standing approval #408 §6 says is the |
| +// only one needed. |
| +func privateStorageFixture(t *testing.T, code string) (*consoleFixture, *objstore.MemProvisioner, string, string) { |
| + t.Helper() |
| + fx, token := instanceFixture(t, code) |
| + pub, priv, _ := sealedFor(t, "irrelevant") |
| + fake := objstore.NewMemProvisioner("eu-west-1") |
| + fx.srv.cfg.SecretsPub = pub |
| + fx.srv.cfg.StoreProvisioner = fake |
| + fx.srv.cfg.StoreDeployment = objstore.Deployment{Backend: "mem", Region: "eu-west-1", BucketPrefix: "carlos-"} |
| + approveStoreAccount(t, fx, "operator@keymail.dev") |
| + return fx, fake, token, priv |
| +} |
| + |
| +// privateManifest is titoManifest carrying the policy. The domain is the |
| +// app's own (not the platform namespace), which keeps the sibling-apex |
| +// branch out of these tests — it has its own coverage. |
| +func privateManifest() provision.Manifest { |
| + m := titoManifest() |
| + m.PrivateStorage = true |
| + return m |
| +} |
| + |
| +func createInstance(t *testing.T, fx *consoleFixture, token, body string) (int, string) { |
| + t.Helper() |
| + status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), token, body) |
| + return status, out |
| +} |
| + |
| +func envStore(t *testing.T, fx *consoleFixture, env string) *objstore.Record { |
| + t.Helper() |
| + rec, _, err := objstore.GetIn(context.Background(), fx.rel, release.OpsAccount, "titogo", env) |
| + if err != nil { |
| + t.Fatalf("read %s store: %v", env, err) |
| + } |
| + return rec |
| +} |
| + |
| +func envVars(t *testing.T, fx *consoleFixture, env string) map[string]appconfig.Var { |
| + t.Helper() |
| + m, err := appconfig.LoadEnvironment(context.Background(), fx.rel, release.OpsAccount, "titogo", env) |
| + if err != nil { |
| + t.Fatalf("read %s bundle: %v", env, err) |
| + } |
| + return m.Vars |
| +} |
| + |
| +// The whole chain, as a signup path drives it: one instance create, and the |
| +// tenant has a bucket of its own whose coordinates are already in the bundle |
| +// the record binds to. No `store create`, no `store provision`, no operator. |
| +func TestInstanceCreateProvisionsPrivateStore(t *testing.T) { |
| + fx, fake, token, priv := privateStorageFixture(t, "code-priv-chain") |
| + enableInstances(t, fx, "titogo", privateManifest()) |
| + |
| + status, out := createInstance(t, fx, token, `{"host":"acme.go.tito.io","region":"ie"}`) |
| + if status != http.StatusCreated { |
| + t.Fatalf("create = %d %s, want 201", status, out) |
| + } |
| + var echoed struct { |
| + Environment string `json:"environment"` |
| + PrivateStoreBucket string `json:"private_store_bucket"` |
| + } |
| + if err := json.Unmarshal([]byte(out), &echoed); err != nil { |
| + t.Fatalf("decode %s: %v", out, err) |
| + } |
| + // The environment is the host's FIRST LABEL, which is the one name |
| + // predictable from the host (spec §6). |
| + if echoed.Environment != "acme" { |
| + t.Fatalf("echoed environment = %q, want acme", echoed.Environment) |
| + } |
| + if echoed.PrivateStoreBucket != "carlos-bab-titogo-acme" { |
| + t.Fatalf("echoed bucket = %q, want carlos-bab-titogo-acme", echoed.PrivateStoreBucket) |
| + } |
| + // The record a box reconciles from carries the binding. |
| + stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if stored.Environment != "acme" { |
| + t.Fatalf("stored environment = %q, want acme", stored.Environment) |
| + } |
| + // The store is real, provisioned, and its own. |
| + rec := envStore(t, fx, "acme") |
| + if rec == nil || rec.Provisioned == nil { |
| + t.Fatalf("acme store = %+v, want a provisioned one", rec) |
| + } |
| + if rec.Provisioned.Bucket != "carlos-bab-titogo-acme" || rec.Provisioned.IAMUser != "carlos-store-bab-titogo-acme" { |
| + t.Fatalf("provisioned = %+v, want the account/app/environment triple", rec.Provisioned) |
| + } |
| + // The grant's authority is the ACCOUNT entitlement, not an operator — |
| + // which is the whole reason no new power was granted to anyone. |
| + if rec.Grant == nil || rec.Grant.Source != objstore.GrantSourceAccountApproval { |
| + t.Fatalf("grant = %+v, want source %q", rec.Grant, objstore.GrantSourceAccountApproval) |
| + } |
| + if _, ok := fake.Buckets["carlos-bab-titogo-acme"]; !ok { |
| + t.Fatalf("backend buckets = %v, want the instance's own", fake.Buckets) |
| + } |
| + // Delivery goes to the INSTANCE's bundle. A credential in the app's |
| + // default bundle would be read by every other instance — the isolation |
| + // failing in exactly the direction that matters. |
| + vars := envVars(t, fx, "acme") |
| + if v, ok := vars["CARLOS_STORE_BUCKET"]; !ok || v.Value != "carlos-bab-titogo-acme" { |
| + t.Fatalf("acme bundle CARLOS_STORE_BUCKET = %+v, want the instance's bucket", v) |
| + } |
| + secret, ok := vars["CARLOS_STORE_SECRET_KEY"] |
| + if !ok || secret.Kind != "secret" { |
| + t.Fatalf("acme bundle CARLOS_STORE_SECRET_KEY = %+v, want a sealed secret", secret) |
| + } |
| + if openSecret(t, priv, secret) == "" { |
| + t.Fatal("the delivered secret did not open") |
| + } |
| + if got := appVars(t, fx, "titogo"); len(got) != 0 { |
| + t.Fatalf("the app's default bundle = %v, want nothing — a shared credential is the isolation failing", got) |
| + } |
| +} |
| + |
| +// An app without the policy is what it was before this branch: a record, no |
| +// store, no bundle, and no backend call. This is the test that fails if the |
| +// policy is ever read as on-by-default. |
| +func TestInstanceCreateWithoutPolicyTouchesNoStore(t *testing.T) { |
| + fx, fake, token, _ := privateStorageFixture(t, "code-priv-off") |
| + enableInstances(t, fx, "titogo", titoManifest()) |
| + |
| + status, out := createInstance(t, fx, token, `{"host":"acme.go.tito.io","region":"ie"}`) |
| + if status != http.StatusCreated { |
| + t.Fatalf("create = %d %s, want 201", status, out) |
| + } |
| + stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if stored.Environment != "" { |
| + t.Fatalf("stored environment = %q, want empty — no policy, no derived binding", stored.Environment) |
| + } |
| + if rec := envStore(t, fx, "acme"); rec != nil { |
| + t.Fatalf("a store was created without the policy: %+v", rec) |
| + } |
| + untouched(t, fake, "a create with no private-storage policy") |
| +} |
| + |
| +// The policy fills a blank; it does not seize the axis. A caller who names |
| +// an environment gets a store for THAT one — which is still the invariant, |
| +// since what is promised is that the record's bundle carries the |
| +// coordinates, not that the bundle has any particular name. |
| +func TestInstanceCreateHonoursAnExplicitEnvironment(t *testing.T) { |
| + fx, _, token, _ := privateStorageFixture(t, "code-priv-explicit") |
| + enableInstances(t, fx, "titogo", privateManifest()) |
| + |
| + status, out := createInstance(t, fx, token, |
| + `{"host":"acme.go.tito.io","region":"ie","environment":"shared-eu"}`) |
| + if status != http.StatusCreated { |
| + t.Fatalf("create = %d %s, want 201", status, out) |
| + } |
| + stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if stored.Environment != "shared-eu" { |
| + t.Fatalf("stored environment = %q, want the one the caller named", stored.Environment) |
| + } |
| + if rec := envStore(t, fx, "shared-eu"); rec == nil || rec.Provisioned == nil { |
| + t.Fatalf("shared-eu store = %+v, want a provisioned one", rec) |
| + } |
| + if rec := envStore(t, fx, "acme"); rec != nil { |
| + t.Fatalf("a store was also made for the derived name: %+v", rec) |
| + } |
| +} |
| + |
| +// A DNS label is not an environment name, and the create door is where that |
| +// has to be said: labels run to 63 characters, environment names to 32, and |
| +// "default" is a legal label that names the app's own bundle. Every refusal |
| +// leaves nothing behind. |
| +func TestInstanceCreateRefusesALabelThatCannotNameAStore(t *testing.T) { |
| + fx, fake, token, _ := privateStorageFixture(t, "code-priv-badlabel") |
| + enableInstances(t, fx, "titogo", privateManifest()) |
| + |
| + for _, label := range []string{ |
| + "default", |
| + "this-label-is-far-too-long-to-be-an-environment-name", |
| + } { |
| + host := label + ".go.tito.io" |
| + status, out := createInstance(t, fx, token, fmt.Sprintf(`{"host":%q,"region":"ie"}`, host)) |
| + if status != http.StatusBadRequest { |
| + t.Errorf("create at %s = %d %s, want 400", host, status, out) |
| + continue |
| + } |
| + if _, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", host); !errors.Is(err, os.ErrNotExist) { |
| + t.Errorf("a refused create at %s left a record (err %v)", host, err) |
| + } |
| + } |
| + untouched(t, fake, "a create whose label cannot name a store") |
| +} |
| + |
| +// An EXPLICIT environment gets the same rails, and the same 400. Without |
| +// this the refusal comes from inside the walk instead — a 502 the caller |
| +// cannot act on, over a name they could have shortened. |
| +func TestInstanceCreateRefusesAnUnnameableExplicitEnvironment(t *testing.T) { |
| + fx, fake, token, _ := privateStorageFixture(t, "code-priv-badenv") |
| + enableInstances(t, fx, "titogo", privateManifest()) |
| + // 32 characters — a legal environment name, and an IAM principal of |
| + // "carlos-store-bab-titogo-" + 32 = 56, under the ceiling. So this is |
| + // the bucket half: "carlos-" + "bab-titogo-" + 32 = 50. Both fit, so |
| + // the case that must refuse is the one the config plane itself refuses. |
| + status, out := createInstance(t, fx, token, |
| + `{"host":"acme.go.tito.io","region":"ie","environment":"UPPER"}`) |
| + if status != http.StatusBadRequest { |
| + t.Fatalf("create with an unusable environment = %d %s, want 400", status, out) |
| + } |
| + if _, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io"); !errors.Is(err, os.ErrNotExist) { |
| + t.Errorf("a refused create left a record (err %v)", err) |
| + } |
| + untouched(t, fake, "a create whose explicit environment cannot name a store") |
| +} |
| + |
| +// The IAM ceiling, and the arithmetic that makes it reachable at all. Both |
| +// names carry the same body: the bucket is <prefix><body> against S3's 63, |
| +// the principal is "carlos-store-" + <body> against IAM's 64. So under any |
| +// prefix longer than "carlos-store-" the BUCKET refusal always fires first, |
| +// and the IAM guard is only ever reached on a short prefix with a long app. |
| +// Encoded here so the guard cannot quietly become dead code. |
| +// |
| +// body = "bab" + "-" + app(25) + "-" + env(25) = 55. |
| +// bucket = "" + 55 = 55, legal. principal = 13 + 55 = 68, one refusal over. |
| +func TestInstanceCreateRefusesAnUnnameableIAMPrincipal(t *testing.T) { |
| + const app = "a-rather-long-application" // 25 |
| + label := strings.Repeat("b", 25) |
| + |
| + fx, token := instanceFixture(t, "code-priv-iamceiling") |
| + if _, err := fx.st.CreateAppIn(OpsAccountID, app, "paul@keymail.dev", time.Now()); err != nil { |
| + t.Fatal(err) |
| + } |
| + pub, _, _ := sealedFor(t, "irrelevant") |
| + fake := objstore.NewMemProvisioner("eu-west-1") |
| + fx.srv.cfg.SecretsPub = pub |
| + fx.srv.cfg.StoreProvisioner = fake |
| + fx.srv.cfg.StoreDeployment = objstore.Deployment{Backend: "mem", Region: "eu-west-1"} |
| + approveStoreAccount(t, fx, "operator@keymail.dev") |
| + enableInstances(t, fx, app, provision.Manifest{ |
| + Schema: 1, App: app, Domain: "go.tito.io", Channel: "stable", PrivateStorage: true}) |
| + |
| + host := label + ".go.tito.io" |
| + status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", app), token, |
| + fmt.Sprintf(`{"host":%q}`, host)) |
| + if status != http.StatusBadRequest { |
| + t.Fatalf("create = %d %s, want 400", status, out) |
| + } |
| + if !strings.Contains(out, "64") { |
| + t.Errorf("refusal %q does not name AWS's ceiling", out) |
| + } |
| + if _, err := provision.GetRecord(context.Background(), fx.rel, "bab", app, host); !errors.Is(err, os.ErrNotExist) { |
| + t.Errorf("a refused create left a record (err %v)", err) |
| + } |
| + untouched(t, fake, "a create whose principal cannot be named") |
| +} |
| + |
| +// A ceiling only the automatic path respected would be a ceiling anyone |
| +// could step over by hand. |
| +func TestStoreCreateRefusedAtTheCeiling(t *testing.T) { |
| + fx, token := storeFixture(t, "code-store-ceiling") |
| + fx.srv.cfg.StoreMaxPerApp = 1 |
| + |
| + if status, body := postRaw(t, storeEnvURL(fx, "titogo", "one"), map[string]any{}, token); status != http.StatusOK { |
| + t.Fatalf("declare one = %d %s", status, body) |
| + } |
| + status, body := postRaw(t, storeEnvURL(fx, "titogo", "two"), map[string]any{}, token) |
| + if status != http.StatusConflict { |
| + t.Fatalf("declare past the cap = %d %s, want 409", status, body) |
| + } |
| + // Re-declaring the one that exists is not a new footprint, so it stands. |
| + if status, body := postRaw(t, storeEnvURL(fx, "titogo", "one"), map[string]any{}, token); status != http.StatusOK { |
| + t.Fatalf("re-declare one = %d %s, want 200", status, body) |
| + } |
| + // And the app's OWN store is not one of the capped collection. |
| + if status, body := postRaw(t, storeURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { |
| + t.Fatalf("declare the app's own store = %d %s, want 200", status, body) |
| + } |
| +} |
| + |
| +// An app that already declared a prefix reads those names in its code. An |
| +// instance store delivering CARLOS_STORE_BUCKET into the same app would be a |
| +// second vocabulary for one thing. |
| +func TestInstanceStoreInheritsTheAppsEnvPrefix(t *testing.T) { |
| + fx, _, token, _ := privateStorageFixture(t, "code-priv-prefix") |
| + enableInstances(t, fx, "titogo", privateManifest()) |
| + if status, out := postRaw(t, storeURL(fx, "titogo"), map[string]any{"env_prefix": "JELLY_S3"}, token); status != http.StatusOK { |
| + t.Fatalf("declare the app store = %d %v", status, out) |
| + } |
| + |
| + if status, out := createInstance(t, fx, token, `{"host":"acme.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| + t.Fatalf("create = %d %s", status, out) |
| + } |
| + vars := envVars(t, fx, "acme") |
| + if _, ok := vars["JELLY_S3_BUCKET"]; !ok { |
| + t.Fatalf("acme bundle = %v, want the app's own JELLY_S3 prefix", varNames(vars)) |
| + } |
| + if _, ok := vars["CARLOS_STORE_BUCKET"]; ok { |
| + t.Fatalf("acme bundle also carries the default prefix: %v", varNames(vars)) |
| + } |
| +} |
| + |
| +func varNames(m map[string]appconfig.Var) []string { |
| + out := make([]string, 0, len(m)) |
| + for k := range m { |
| + out = append(out, k) |
| + } |
| + return out |
| +} |
| + |
| +// Declaring the policy is refused in the two states where it could only ever |
| +// be a trap — a deployment with no object storage, and an account nobody |
| +// approved. It fails once here, to the person, instead of once per signup in |
| +// a door whose caller is a machine. |
| +func TestEnableRefusesAPolicyItCannotHonour(t *testing.T) { |
| + t.Run("no backend", func(t *testing.T) { |
| + fx, _, token, _ := privateStorageFixture(t, "code-priv-enable-nobackend") |
| + fx.srv.cfg.AppsDomain = "oncarlos.test" |
| + fx.srv.cfg.StoreProvisioner = nil |
| + status, _, out := rawReq(t, http.MethodPut, enableURL(fx, "bab", "titogo"), token, `{"private_storage":true}`) |
| + if status != http.StatusServiceUnavailable { |
| + t.Fatalf("enable = %d %s, want 503", status, out) |
| + } |
| + if !strings.Contains(out, "CARLOS_STORE_BACKEND") { |
| + t.Errorf("refusal %q does not name the variable", out) |
| + } |
| + assertNoPolicy(t, fx) |
| + }) |
| + t.Run("unapproved account", func(t *testing.T) { |
| + fx, token := instanceFixture(t, "code-priv-enable-unapproved") |
| + fx.srv.cfg.AppsDomain = "oncarlos.test" |
| + fx.srv.cfg.StoreProvisioner = objstore.NewMemProvisioner("eu-west-1") |
| + status, _, out := rawReq(t, http.MethodPut, enableURL(fx, "bab", "titogo"), token, `{"private_storage":true}`) |
| + if status != http.StatusForbidden { |
| + t.Fatalf("enable = %d %s, want 403", status, out) |
| + } |
| + if !strings.Contains(out, "carlos store apply") { |
| + t.Errorf("refusal %q does not name the way out", out) |
| + } |
| + assertNoPolicy(t, fx) |
| + }) |
| +} |
| + |
| +func assertNoPolicy(t *testing.T, fx *consoleFixture) { |
| + t.Helper() |
| + m, err := provision.GetManifest(context.Background(), fx.rel, "bab", "titogo") |
| + if err == nil && m.PrivateStorage { |
| + t.Fatal("a refused enable wrote the policy anyway") |
| + } |
| +} |
| + |
| +// Turning it OFF is never refused, and the asymmetry is the point: an |
| +// account whose entitlement was later declined must be able to get its |
| +// create door working again without waiting on an operator. |
| +func TestEnableTurningPrivateStorageOffIsNeverRefused(t *testing.T) { |
| + fx, token := instanceFixture(t, "code-priv-enable-off") |
| + fx.srv.cfg.AppsDomain = "oncarlos.test" |
| + enableInstances(t, fx, "titogo", provision.Manifest{ |
| + Schema: 1, App: "titogo", Domain: "bab.oncarlos.test", PrivateStorage: true}) |
| + // No backend, no entitlement — the state the ON path refuses in. |
| + fx.srv.cfg.StoreProvisioner = nil |
| + |
| + status, _, out := rawReq(t, http.MethodPut, enableURL(fx, "bab", "titogo"), token, `{"private_storage":false}`) |
| + if status != http.StatusOK { |
| + t.Fatalf("enable off = %d %s, want 200", status, out) |
| + } |
| + m, err := provision.GetManifest(context.Background(), fx.rel, "bab", "titogo") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if m.PrivateStorage { |
| + t.Fatal("the policy survived being turned off") |
| + } |
| +} |
| + |
| +// An enable that never mentions storage preserves the policy — the same |
| +// absent-vs-explicit split channel and seed already have. A `carlos |
| +// instances enable -channel beta` that silently unset a tenant's isolation |
| +// is the failure this pins. |
| +func TestEnablePreservesPrivateStorage(t *testing.T) { |
| + fx, _, token, _ := privateStorageFixture(t, "code-priv-enable-preserve") |
| + fx.srv.cfg.AppsDomain = "oncarlos.test" |
| + enableInstances(t, fx, "titogo", provision.Manifest{ |
| + Schema: 1, App: "titogo", Domain: "bab.oncarlos.test", PrivateStorage: true}) |
| + |
| + status, _, out := rawReq(t, http.MethodPut, enableURL(fx, "bab", "titogo"), token, `{"channel":"beta"}`) |
| + if status != http.StatusOK { |
| + t.Fatalf("enable = %d %s, want 200", status, out) |
| + } |
| + var res struct { |
| + Channel string `json:"channel"` |
| + PrivateStorage bool `json:"private_storage"` |
| + } |
| + if err := json.Unmarshal([]byte(out), &res); err != nil { |
| + t.Fatal(err) |
| + } |
| + if res.Channel != "beta" || !res.PrivateStorage { |
| + t.Fatalf("reply = %+v, want channel beta with the policy intact", res) |
| + } |
| + m, err := provision.GetManifest(context.Background(), fx.rel, "bab", "titogo") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !m.PrivateStorage { |
| + t.Fatal("a channel-only enable dropped the private-storage policy") |
| + } |
| +} |
| + |
| +// Declaring the policy stays a PERSON's act — the wall that makes this a |
| +// better answer than widening the instance role. An app token creates |
| +// instances all day and cannot write the manifest they are created under. |
| +func TestPrivateStoragePolicyStaysAHumanAct(t *testing.T) { |
| + fx, _, token, _ := privateStorageFixture(t, "code-priv-human") |
| + fx.srv.cfg.AppsDomain = "oncarlos.test" |
| + appToken := mintInstanceToken(t, fx, token, "titogo") |
| + |
| + status, _, out := rawReq(t, http.MethodPut, enableURL(fx, "bab", "titogo"), appToken, `{"private_storage":true}`) |
| + if status != http.StatusForbidden && status != http.StatusUnauthorized { |
| + t.Fatalf("app token enable = %d %s, want a refusal", status, out) |
| + } |
| + assertNoPolicy(t, fx) |
| +} |
| + |
| +// mintInstanceToken is `carlos instances token` — the service credential a |
| +// signup path holds. |
| +func mintInstanceToken(t *testing.T, fx *consoleFixture, personToken, app string) string { |
| + t.Helper() |
| + u := fx.ts.URL + "/api/cli/apps/bab/" + app + "/instance-token" |
| + status, _, out := rawReq(t, http.MethodPost, u, personToken, `{}`) |
| + if status != http.StatusCreated { |
| + t.Fatalf("mint instance token = %d %s", status, out) |
| + } |
| + var res struct { |
| + Token string `json:"token"` |
| + } |
| + if err := json.Unmarshal([]byte(out), &res); err != nil { |
| + t.Fatal(err) |
| + } |
| + return res.Token |
| +} |
| + |
| +// The ceiling in force, and the two ways a deployment names it. Zero in the |
| +// config is "not configured", never "no stores allowed" — an unset variable |
| +// turning an app into one that can hold no store at all is the wrong |
| +// direction for a default to fail in. |
| +// |
| +// The number itself is pinned because it is a JUDGEMENT about AWS's quota |
| +// rather than an arithmetic consequence of anything: IAM allows 5,000 users |
| +// per cloud account, so one app's share is a fifth of that. An earlier draft |
| +// said 100, which would have refused an ordinary managed product at its |
| +// hundredth customer while protecting nothing (James, 2026-09-23). A future |
| +// change to the constant should be a decision someone took, not a drift. |
| +func TestStoreCeilingDefault(t *testing.T) { |
| + s := &Server{} |
| + if got := s.storeMaxPerApp(); got != 1000 { |
| + t.Errorf("unconfigured ceiling = %d, want 1000", got) |
| + } |
| + s.cfg.StoreMaxPerApp = 25 |
| + if got := s.storeMaxPerApp(); got != 25 { |
| + t.Errorf("configured ceiling = %d, want 25", got) |
| + } |
| +} |
diff --git a/internal/console/apistore.go b/internal/console/apistore.go| index 79a8fced7..07019da19 100644 |
| --- a/internal/console/apistore.go |
| +++ b/internal/console/apistore.go |
| @@ -398,6 +398,17 @@ func (s *Server) handleAPIStoreCreate(w http.ResponseWriter, r *http.Request) { |
| } |
| ctx, cancel := context.WithTimeout(r.Context(), storeTimeout) |
| defer cancel() |
| + // The environment-store ceiling (issue #412 §8) is enforced HERE as well |
| + // as on the automatic path. A cap that only the create-an-instance door |
| + // respected would be a cap anyone could step over by hand, and the |
| + // footprint it bounds — a bucket and an IAM user per store, against |
| + // IAM's per-account limits — does not care which door minted it. |
| + // Re-declaring a store that already exists is never capped; storeCeiling |
| + // carries that carve-out. |
| + if ref := s.storeCeiling(ctx, acct, app, env); ref != nil { |
| + jsonError(w, ref.Status, ref.Msg) |
| + return |
| + } |
| rec, etag, err := objstore.GetIn(ctx, s.cfg.Releases, acct.Sqid, app, env) |
| if err != nil { |
| // Logged, not returned: a store error carries bucket names, endpoints |
diff --git a/internal/console/instanceprivatestore.go b/internal/console/instanceprivatestore.go| new file mode 100644 |
| index 000000000..b5a9d10aa |
| --- /dev/null |
| +++ b/internal/console/instanceprivatestore.go |
| @@ -0,0 +1,312 @@ |
| +// The app-declared private-storage policy (issue #412, spec |
| +// 2026-09-22-instance-private-storage-design.md): an app may say once, on |
| +// its instances manifest, that every instance it creates gets a config |
| +// environment of its own and an object store bound to that environment. |
| +// |
| +// The whole point is the SPLIT. Declaring the policy is |
| +// `handleAPIInstanceEnable` — person-only, zero Set, no service credential |
| +// reaches it. Spending it is `handleAPIInstanceCreate`, which an |
| +// `instance`-role credential can already reach. So a public signup path |
| +// provisions a tenant's own bucket without ever holding the power to create |
| +// one, which is what makes this a better answer than widening the role |
| +// (#410, which this makes unnecessary). |
| +// |
| +// Nothing here is a second implementation of provisioning. The walk this |
| +// file drives is the one `handleAPIStoreProvision` drives, with the same |
| +// lock, the same seal key, the same timeout and the same |
| +// GrantSourceAccountApproval — factored so the automatic path and the |
| +// member's own `carlos store create --environment` cannot drift. |
| +package console |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "fmt" |
| + "log" |
| + "net/http" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/appconfig" |
| + "github.com/carlosframework/platform/internal/objstore" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// storeMaxPerAppDefault is the ceiling on ENVIRONMENT stores under one app |
| +// when the deployment names none (spec §8, issue #412's point 3). |
| +// |
| +// A bucket plus an IAM user per instance is a bigger footprint than an |
| +// instance, and the binding limit is IAM's: 5,000 users per cloud account, |
| +// an adjustable quota |
| +// (https://docs.aws.amazon.com/general/latest/gr/iam-service.html). So a |
| +// signup path that can mint them needs a bar it stops at — but the bar has |
| +// to be sized against that number rather than picked for looking tidy. An |
| +// earlier draft said 100, which was a fifth of one app's fair share of the |
| +// quota and would have refused a perfectly ordinary managed product at its |
| +// hundredth customer (James, 2026-09-23). |
| +// |
| +// Per APP rather than per account, and that is a cost decision stated as |
| +// such: the count is one prefix listing of the app, which the create door |
| +// can afford on every request. A deployment-wide count would mean listing |
| +// every app's stores on every create. |
| +// |
| +// WHAT THIS DOES NOT BOUND, said plainly because the per-app shape invites |
| +// the wrong reading: five apps at this default reach 5,000 between them, and |
| +// the deployment's own IAM principals are not in that sum. This is a brake |
| +// on ONE runaway signup path, not a budget for the cloud account. A real |
| +// account-wide bound needs a counter kept on the side rather than one |
| +// derived by listing, and it is not here. |
| +const storeMaxPerAppDefault = 1000 |
| + |
| +// storeMaxPerApp is the ceiling in force. Zero in the config means "not |
| +// configured", not "no stores allowed": a cap of zero would turn an |
| +// unset variable into an app that can have no environment store at all. |
| +func (s *Server) storeMaxPerApp() int { |
| + if s.cfg.StoreMaxPerApp > 0 { |
| + return s.cfg.StoreMaxPerApp |
| + } |
| + return storeMaxPerAppDefault |
| +} |
| + |
| +// storeCeiling refuses an app that has reached its environment-store cap. |
| +// |
| +// env is the environment ABOUT to be given a store, and a name that already |
| +// has one is never refused: the cap is on the FOOTPRINT, and re-declaring |
| +// an existing store adds no bucket and no principal. Without that carve-out |
| +// a retry of a create that failed just after provisioning would be turned |
| +// away by the very ceiling it had reached, with no way forward at all. |
| +// |
| +// A listing failure is a refusal, not a pass. This is a spend gate; reading |
| +// "how many are there already" as "none" on a bad round trip is the one |
| +// error direction that costs money. |
| +func (s *Server) storeCeiling(ctx context.Context, acct *Account, app, env string) *doorRefusal { |
| + if env == "" { |
| + return nil // the app's own store is not one of the capped collection |
| + } |
| + envs, err := objstore.StoreEnvironments(ctx, s.cfg.Releases, acct.Sqid, app) |
| + if err != nil { |
| + log.Printf("console: store ceiling %s/%s: %v", acct.Sqid, app, err) |
| + return &doorRefusal{Status: http.StatusBadGateway, |
| + Msg: "could not count this app's stores — try again"} |
| + } |
| + for _, e := range envs { |
| + if e == env { |
| + return nil |
| + } |
| + } |
| + max := s.storeMaxPerApp() |
| + if len(envs) < max { |
| + return nil |
| + } |
| + return &doorRefusal{Status: http.StatusConflict, |
| + Msg: fmt.Sprintf("%s already has %d environment stores, which is this deployment's limit of %d — destroy one in the console, or ask an operator to raise CARLOS_STORE_MAX_PER_APP", |
| + app, len(envs), max)} |
| +} |
| + |
| +// instanceStoreNames refuses an environment that cannot carry a store, for |
| +// an app under the policy (spec §6). |
| +// |
| +// derived says where the name came from — the host's first label, or the |
| +// caller's own `environment` — and it changes only the sentence, never the |
| +// rails. The first label is the derived name because it is the one |
| +// predictable from the host, already unique within the app (the record key |
| +// IS the host), and readable in `store status` at a hundred of them, which |
| +// is #412's point 1. |
| +// |
| +// Three refusals, all BEFORE anything is written, because each of them is |
| +// otherwise discovered somewhere worse: |
| +// |
| +// - a DNS label is not an environment name. Labels run to 63 characters |
| +// and environment names to 32, and "default" is a perfectly legal label |
| +// that names the app's own bundle rather than a named overlay. |
| +// - the bucket name is S3's 63. The walk checks this itself, but reaching |
| +// it there means an instance record already written over an estate that |
| +// cannot exist — and the walk's own refusal arrives as a 502 the caller |
| +// cannot act on, not a 400 naming the part to shorten. |
| +// - the principal name is IAM's 64, for the same reason. An environment |
| +// store's name carries three components, so a long enough (account, |
| +// app, environment) is simply unnameable. |
| +// |
| +// Each sentence names the part the caller can shorten, because from a |
| +// signup path the host label is usually the only one they control. |
| +func (s *Server) instanceStoreNames(acct *Account, app, env string, derived bool) *doorRefusal { |
| + source := "the environment you named" |
| + fix := "choose a shorter environment name" |
| + if derived { |
| + source = "the host's first label" |
| + fix = "choose a different host label" |
| + } |
| + if !appconfig.ValidEnvironmentName(env) { |
| + return &doorRefusal{Status: http.StatusBadRequest, |
| + Msg: fmt.Sprintf("%s gives every instance its own storage, which needs %s to be a usable environment name — %q is not (want ^[a-z0-9-]{1,32}$, and not \"default\"). Please %s", app, source, env, fix)} |
| + } |
| + bucket := objstore.MintedBucketIn(s.cfg.StoreDeployment, acct.Sqid, app, env) |
| + if !objstore.ValidBucketName(bucket) { |
| + return &doorRefusal{Status: http.StatusBadRequest, |
| + Msg: fmt.Sprintf("this instance's bucket would be named %q, which is %d characters and not a legal S3 name — %s", bucket, len(bucket), fix)} |
| + } |
| + user := objstore.UserNameIn(acct.Sqid, app, env) |
| + if !objstore.ValidUserName(user) { |
| + return &doorRefusal{Status: http.StatusBadRequest, |
| + Msg: fmt.Sprintf("this instance's IAM user would be named %q, which is %d characters, over AWS's %d — %s, or shorten the app name", user, len(user), objstore.MaxIAMUserName, fix)} |
| + } |
| + return nil |
| +} |
| + |
| +// instanceStoreProvision declares and provisions one environment's store |
| +// under the account's standing entitlement, and answers w itself on every |
| +// refusal. The caller has already proved membership and spent its write |
| +// budget; what is left is the store plane's own gates, in the order |
| +// handleAPIStoreProvision applies them. |
| +// |
| +// actor is the create door's principal — a person's address or the service |
| +// credential's `app:<sqid>/<app>` audit actor — and it lands in the grant |
| +// block, so a bucket minted by a signup path names the signup path. |
| +// |
| +// The env prefix is INHERITED from the app's own store when it has one. An |
| +// app that declared `carlos store create --env-prefix JELLY_S3` reads those |
| +// names in its code, and an instance store delivering CARLOS_STORE_BUCKET |
| +// into the same app would be a second vocabulary for one thing. With no app |
| +// store declared, the default prefix is taken — what a fresh declaration |
| +// would have taken anyway. |
| +// |
| +// ctx must be the REQUEST context, not a short-deadline derivative of it: |
| +// the walk below takes its own storeGrantTimeout, which is longer than the |
| +// create door's own deadline, and a walk that inherited a fifteen-second |
| +// budget would abandon a bucket it had already paid for. |
| +func (s *Server) instanceStoreProvision(ctx context.Context, w http.ResponseWriter, acct *Account, app, env, actor string) (*objstore.Record, bool) { |
| + if s.cfg.StoreProvisioner == nil { |
| + // The manifest carries a policy this deployment cannot honour. The |
| + // enable door refuses to write one in this state (§5), so reaching |
| + // here means the backend was unwired afterwards — a configuration |
| + // fact, named as one. |
| + jsonError(w, http.StatusServiceUnavailable, |
| + "object storage is not configured for this deployment (CARLOS_STORE_BACKEND), and "+app+" gives every instance its own storage — no instance can be created until one is wired or the policy is turned off (carlos instances enable --private-storage=false)") |
| + return nil, false |
| + } |
| + preCtx, preCancel := context.WithTimeout(ctx, storeTimeout) |
| + defer preCancel() |
| + if _, ok := s.storeProvisionApproval(preCtx, w, acct); !ok { |
| + return nil, false |
| + } |
| + pub, ok := s.storeSealKey(w) |
| + if !ok { |
| + return nil, false |
| + } |
| + // The same lock handleAPIStoreProvision takes, keyed the same way, so a |
| + // signup creating an instance and a member running `carlos store create |
| + // --environment` against the same name serialise against each other. |
| + defer s.storeWalks.lock(acct.Sqid, app, env)() |
| + // The walk's deadline starts AFTER the lock, for the reason that door |
| + // states: a request that queued behind a full-length walk would |
| + // otherwise reach the backend holding a spent context. |
| + walkCtx, cancel := context.WithTimeout(ctx, storeGrantTimeout) |
| + defer cancel() |
| + // Re-read under the lock: a queued request may have waited while an |
| + // operator declined the account. |
| + e, ok := s.storeProvisionApproval(walkCtx, w, acct) |
| + if !ok { |
| + return nil, false |
| + } |
| + // The declaration the walk requires. GrantIn refuses a store nobody |
| + // declared (ErrNotDeclared) because the env prefix is the member's |
| + // choice and a store with nowhere to be delivered is not one to invent |
| + // — under this policy the APP made that choice, once, on the manifest, |
| + // so the declaration is written here rather than demanded of a caller |
| + // who has no verb to write it with. |
| + prefix, ok := s.storeInheritedPrefix(walkCtx, w, acct, app) |
| + if !ok { |
| + return nil, false |
| + } |
| + rec, etag, err := objstore.GetIn(walkCtx, s.cfg.Releases, acct.Sqid, app, env) |
| + if err != nil { |
| + log.Printf("console: instance store read %s/%s/%s: %v", acct.Sqid, app, env, err) |
| + jsonError(w, http.StatusBadGateway, "could not read this instance's store record — try again") |
| + return nil, false |
| + } |
| + if rec == nil { |
| + rec = &objstore.Record{} |
| + } |
| + // Only when absent. A re-run over an existing declaration must not |
| + // re-stamp its author and timestamp, and must not move a prefix the app |
| + // is already reading — that is the one rewrite handleAPIStoreCreate |
| + // refuses with a 409, and this path has no member to show it to. |
| + if rec.Declared == nil { |
| + rec.Declared = &objstore.Declared{EnvPrefix: prefix, At: time.Now().UTC(), By: actor} |
| + if _, err := objstore.PutIn(walkCtx, s.cfg.Releases, acct.Sqid, app, env, rec, etag); err != nil { |
| + if errors.Is(err, release.ErrCASConflict) { |
| + jsonError(w, http.StatusConflict, |
| + "this instance's store record changed while it was being created — try again") |
| + return nil, false |
| + } |
| + log.Printf("console: instance store declare %s/%s/%s: %v", acct.Sqid, app, env, err) |
| + jsonError(w, http.StatusBadGateway, "could not declare this instance's store — nothing was created; try again") |
| + return nil, false |
| + } |
| + } |
| + out, err := objstore.GrantIn(walkCtx, s.cfg.Releases, s.cfg.StoreProvisioner, s.cfg.StoreDeployment, |
| + pub, acct.Sqid, app, env, e.Decision.By, objstore.GrantSourceAccountApproval, "", time.Now()) |
| + if err != nil { |
| + // Logged, not returned: a walk error carries bucket names, IAM ARNs |
| + // and request ids. The sentence the caller gets is the one they can |
| + // act on, and it is honest about what is now true — the declaration |
| + // may stand, the instance does not, and a retry of the SAME host |
| + // resumes this exact store because its name came from the host. |
| + log.Printf("console: instance store provision %s/%s/%s: %v", acct.Sqid, app, env, err) |
| + jsonError(w, http.StatusBadGateway, |
| + "could not provision this instance's storage, so the instance was not created — run the same command again to resume it") |
| + return nil, false |
| + } |
| + // Audited from here as well as from the store doors, because this is the |
| + // one path where a bucket is minted by a caller that is not a person: |
| + // an account reading its own rows has to be able to see that its signup |
| + // path spent money, and by whom it was authorised. |
| + s.audit(acct, app, actor, "store-provision", |
| + "provisioned "+storeBucketName(out)+" for instance environment "+env+" (instance create)") |
| + return out, true |
| +} |
| + |
| +// storeInheritedPrefix reads the app's own store for the variable-name |
| +// prefix a new environment store should deliver under. Absent is not an |
| +// error — an app with no store of its own is the ordinary shape for a |
| +// private-storage app, and takes the default. |
| +func (s *Server) storeInheritedPrefix(ctx context.Context, w http.ResponseWriter, acct *Account, app string) (string, bool) { |
| + rec, _, err := objstore.Get(ctx, s.cfg.Releases, acct.Sqid, app) |
| + if err != nil { |
| + log.Printf("console: instance store prefix %s/%s: %v", acct.Sqid, app, err) |
| + jsonError(w, http.StatusBadGateway, "could not read this app's store settings — try again") |
| + return "", false |
| + } |
| + return storeEffectivePrefix(rec), true |
| +} |
| + |
| +// instancePrivateStorageAllowed is the enable door's half: the two states in |
| +// which declaring the policy would be a trap rather than a policy (spec §5). |
| +// |
| +// A manifest that says "every instance gets a bucket" on a deployment with |
| +// no object storage, or in an account nobody approved, does not fail when it |
| +// is written. It fails once per signup, afterwards, in a door whose caller |
| +// is a machine — and the two sentences below are the only two that name the |
| +// fix. So they are said here, to the person, at the moment they declare it. |
| +// |
| +// Turning the policy OFF is never refused, and that asymmetry is the point: |
| +// an account whose entitlement was later declined must be able to get its |
| +// create door working again without an operator. |
| +func (s *Server) instancePrivateStorageAllowed(ctx context.Context, w http.ResponseWriter, acct *Account) bool { |
| + if s.cfg.StoreProvisioner == nil { |
| + jsonError(w, http.StatusServiceUnavailable, |
| + "object storage is not configured for this deployment (CARLOS_STORE_BACKEND) — an app cannot be given per-instance storage here") |
| + return false |
| + } |
| + e, _, err := objstore.GetEntitlement(ctx, s.cfg.Releases, acct.Sqid) |
| + if err != nil { |
| + log.Printf("console: instances enable entitlement %s: %v", acct.Sqid, err) |
| + jsonError(w, http.StatusBadGateway, "could not read this account's storage approval — try again") |
| + return false |
| + } |
| + if !e.Approved() { |
| + jsonError(w, http.StatusForbidden, |
| + "this account is not approved for object storage, so its instances cannot be given storage of their own — run `carlos store apply` and ask an operator to approve it first") |
| + return false |
| + } |
| + return true |
| +} |
diff --git a/internal/console/server.go b/internal/console/server.go| index 7c1aedd22..ed4352157 100644 |
| --- a/internal/console/server.go |
| +++ b/internal/console/server.go |
| @@ -592,6 +592,17 @@ type Config struct { |
| // lives — a walk that guessed would create the next bucket somewhere |
| // else, and the record would say so afterwards. |
| StoreDeployment objstore.Deployment |
| + // StoreMaxPerApp caps how many ENVIRONMENT stores one app may have |
| + // (CARLOS_STORE_MAX_PER_APP; 0 = storeMaxPerAppDefault). The app's own |
| + // store is not one of them and is never capped. |
| + // |
| + // It exists because #412 made a store something a signup path can mint: |
| + // a bucket plus an IAM user per instance is a bigger footprint than an |
| + // instance, and IAM's limits are per cloud account, so the automatic |
| + // path needs a bar it stops at. Enforced on the MANUAL door too |
| + // (handleAPIStoreCreate) — a ceiling only the automatic path respected |
| + // would be a ceiling anyone could step over by hand. |
| + StoreMaxPerApp int |
| // StoreOperators are the addresses allowed to drive the GRANT door, and |
| // only that one (CARLOS_STORE_OPERATORS). Its own list rather than |
| // console ownership on purpose: granting CREATES backend estate and |
diff --git a/internal/provision/provision.go b/internal/provision/provision.go| index 7c68ed826..e958c2dc8 100644 |
| --- a/internal/provision/provision.go |
| +++ b/internal/provision/provision.go |
| @@ -190,6 +190,26 @@ type Manifest struct { |
| Channel string `json:"channel,omitempty"` |
| Regions []string `json:"regions,omitempty"` |
| Seed []string `json:"seed,omitempty"` |
| + |
| + // PrivateStorage is the app's standing policy that every instance |
| + // created under this manifest gets a config environment of its own and |
| + // an object store bound to that environment (issue #412, spec |
| + // 2026-09-22-instance-private-storage-design.md §5). Declared once, by |
| + // a signed-in person, at `carlos instances enable --private-storage`; |
| + // spent by the create door, which an `instance`-role service credential |
| + // can already reach. That split is the point: the policy widens no |
| + // role, so a signup path provisions a tenant's bucket without ever |
| + // holding the power to create one. |
| + // |
| + // A plain bool, not a pointer, and the difference from Record's |
| + // Hibernate and StatusNamed is worth stating because their reason does |
| + // not apply here. Those are CONVERGED columns — a box re-reads them |
| + // every pass, so "absent" has to be distinguishable from an explicit |
| + // false or the first pass after deploy acts on a value nobody wrote. |
| + // This is read at one door, at create time, and false IS the behaviour |
| + // every manifest in a production bucket already has. There is nothing |
| + // for absence to mean that false does not already mean. |
| + PrivateStorage bool `json:"private_storage,omitempty"` |
| } |
| // ManifestKey, RecordsPrefix and RecordKey are the bucket key conventions |
diff --git a/internal/provision/provision_test.go b/internal/provision/provision_test.go| index 1ab77449c..e233f5bce 100644 |
| --- a/internal/provision/provision_test.go |
| +++ b/internal/provision/provision_test.go |
| @@ -1115,3 +1115,50 @@ func TestLayoutJournalPath(t *testing.T) { |
| t.Errorf("LayoutJournalPath = %q", got) |
| } |
| } |
| + |
| +// PrivateStorage is additive and its ZERO VALUE is the legacy behaviour: a |
| +// manifest written before the field existed — every manifest in a |
| +// production bucket — must read as an app whose instances share the app's |
| +// storage, which is what those apps do (issue #412, spec §5). |
| +func TestManifestPrivateStorageIsAdditive(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + |
| + // The bytes an older console wrote: no such key at all. |
| + legacy := []byte(`{"schema":1,"app":"tito","domain":"go.tito.io","channel":"stable"}`) |
| + if _, err := s.Put(ctx, ManifestKey("bad", "tito"), legacy, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + got, err := GetManifest(ctx, s, "bad", "tito") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if got.PrivateStorage { |
| + t.Fatal("a manifest written before the field reads as private-storage — every app in production would start minting buckets") |
| + } |
| + |
| + got.PrivateStorage = true |
| + if err := PutManifest(ctx, s, "bad", "tito", *got); err != nil { |
| + t.Fatal(err) |
| + } |
| + back, err := GetManifest(ctx, s, "bad", "tito") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !back.PrivateStorage { |
| + t.Fatalf("policy did not survive a round trip: %+v", back) |
| + } |
| + // omitempty: an app without the policy writes no key, so nothing an old |
| + // reader parses changes shape. |
| + back.PrivateStorage = false |
| + if err := PutManifest(ctx, s, "bad", "tito", *back); err != nil { |
| + t.Fatal(err) |
| + } |
| + raw, _, err := s.Get(ctx, ManifestKey("bad", "tito")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if strings.Contains(string(raw), "private_storage") { |
| + t.Errorf("an app without the policy still writes the key: %s", raw) |
| + } |
| +} |