Gate the quantity worker on sandbox proof of its fence; fix the orphan bucket the simulation found
Storage: - A grant over an unfinished teardown voids that teardown's note that the bucket was deleted, before it does anything that could bring the bucket back. Without this a grant that recreated the bucket and then failed left the resumed teardown believing there was nothing to delete, and the bucket stayed with nobody holding it. Found by the simulation at seed 1085. - The simulation's seed 778 failure was its own bookkeeping: a teardown refused as busy did nothing, and the model forgot the app's retained state. - A test that a teardown stops when the ledger drops its decision mid-walk. - 30,000 seeds pass. Quantity worker: - A second switch. The worker runs only with a proration policy AND CONSOLE_BILLING_QUANTITY_FENCE=stripe-sandbox-verified. The fence rests on Stripe's documented idempotency rules, which the tests model with a fake. That is not proof, so the worker cannot start on the policy decision alone. - TestSandboxQuantityFence: the check against Stripe test mode that the second switch waits on. Skipped without a test-mode restricted key. Not run. - A quantity that goes down inside a period files billing_quantity_lowered. Before this the console followed the provider's number down with no review, and a comment claimed otherwise. - The fence's limit is written down where the fence is: a consumed key is not a permanent token, Stripe forgets keys after 24 hours, and a request that first arrives after that runs. Detected and held, not prevented. Documents: - docs/superpowers/specs/2026-10-01-stripe-idempotency-evidence.md: Stripe's documentation quoted, mapped to the code, with what is unverified. - Round six reports and dispositions. Path A of the attestation plan is parked and replication-write completeness is recorded as a launch blocker. - implementation/carloku-paid-launch/OUTSTANDING.md: what is reviewable, what is tested, what is proven, and every launch blocker. - The round seven review brief.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
20 files changed,
+1304
−28
cmd/console/billing.go+7 −0cmd/console/billing_test.go+22 −0docs/superpowers/plans/2026-10-01-durability-attestation-path.md+18 −1docs/superpowers/specs/2026-10-01-stripe-idempotency-evidence.md+113 −0docs/superpowers/specs/CODEX-DISPOSITIONS-carloku-paid-launch-overnight.md+59 −0docs/superpowers/specs/CODEX-FINDINGS-carloku-paid-launch-overnight-round6.md+227 −0docs/superpowers/specs/CODEX-REVIEW-BRIEF-carloku-paid-launch-overnight-round7.md+33 −0docs/superpowers/specs/OPUS-REVIEW-carloku-paid-launch-overnight-round6.md+185 −0implementation/carloku-paid-launch/AGENT_HANDOFF.md+10 −7implementation/carloku-paid-launch/OUTSTANDING.md+70 −0implementation/carloku-paid-launch/requirements-audit.json+9 −6implementation/carloku-paid-launch/stripe-provisioning.md+9 −4internal/billingstripe/sandbox_fence_test.go+292 −0internal/console/billingquantity.go+44 −7internal/console/billingquantity_test.go+60 −1internal/console/billingservice.go+28 −0internal/objstore/objstore.go+4 −1internal/objstore/ownership_test.go+61 −0internal/objstore/walk.go+15 −0internal/objstore/walk_sim_test.go+38 −1
diff --git a/cmd/console/billing.go b/cmd/console/billing.go| index 12a633a47..4beaab333 100644 |
| --- a/cmd/console/billing.go |
| +++ b/cmd/console/billing.go |
| @@ -29,10 +29,17 @@ func billingConfig(getenv func(string) string) (*console.BillingConfig, error) { |
| // The proration collection policy is a product decision with no |
| // default: unset leaves member additions pending. |
| Proration: getenv("CONSOLE_BILLING_PRORATION"), |
| + // The quantity worker's second switch: set only after the fence it |
| + // relies on has been verified against a Stripe sandbox. Unset, the |
| + // worker stays off whatever the proration policy says. |
| + QuantityFence: getenv("CONSOLE_BILLING_QUANTITY_FENCE"), |
| } |
| if cfg.Proration != "" && cfg.Proration != "immediate" && cfg.Proration != "next_invoice" { |
| return nil, errBillingConfiguration |
| } |
| + if cfg.QuantityFence != "" && cfg.QuantityFence != "stripe-sandbox-verified" { |
| + return nil, errBillingConfiguration |
| + } |
| prefix := "rk_test_" |
| if cfg.LiveMode { |
| prefix = "rk_live_" |
diff --git a/cmd/console/billing_test.go b/cmd/console/billing_test.go| index 19813df88..56c937133 100644 |
| --- a/cmd/console/billing_test.go |
| +++ b/cmd/console/billing_test.go |
| @@ -175,3 +175,25 @@ func TestBillingConfigWebhookSecretRotation(t *testing.T) { |
| } |
| } |
| } |
| + |
| +// The quantity worker's second switch is read as it is written: unset by |
| +// default, one accepted value, and anything else refuses to start rather |
| +// than being taken for either. |
| +func TestBillingConfigQuantityFenceSwitch(t *testing.T) { |
| + env := billingEnvFixture(t) |
| + get := func(key string) string { return env[key] } |
| + cfg, err := billingConfig(get) |
| + if err != nil || cfg.QuantityFence != "" { |
| + t.Fatalf("default: fence %q err %v, want unset", cfg.QuantityFence, err) |
| + } |
| + env["CONSOLE_BILLING_QUANTITY_FENCE"] = "stripe-sandbox-verified" |
| + if cfg, err = billingConfig(get); err != nil || cfg.QuantityFence != "stripe-sandbox-verified" { |
| + t.Fatalf("verified: fence %q err %v", cfg.QuantityFence, err) |
| + } |
| + for _, bad := range []string{"verified", "true", "1", "stripe-sandbox-verified "} { |
| + env["CONSOLE_BILLING_QUANTITY_FENCE"] = bad |
| + if _, err := billingConfig(get); !errors.Is(err, errBillingConfiguration) { |
| + t.Fatalf("fence value %q accepted", bad) |
| + } |
| + } |
| +} |
diff --git a/docs/superpowers/plans/2026-10-01-durability-attestation-path.md b/docs/superpowers/plans/2026-10-01-durability-attestation-path.md| index 63ba421f9..ff62f6a11 100644 |
| --- a/docs/superpowers/plans/2026-10-01-durability-attestation-path.md |
| +++ b/docs/superpowers/plans/2026-10-01-durability-attestation-path.md |
| @@ -1,6 +1,6 @@ |
| # Replication writes: how a box could honestly attest |
| -Date: 2026-10-01. Status: the meter folds and holds; no production box attests; nothing here flips that. Path A is chosen and built end to end locally: the per-account fold, the scope, the reader and the agent's call (see "Decision and progress"). The call is behind a switch that is off everywhere. |
| +Date: 2026-10-01. Status: the meter folds and holds; no production box attests; nothing here flips that. **Path A (scoped attestation) is parked: round six showed it cannot prove an exclusive writer. Its code is off the launch branch and kept on `feat/carloku-scoped-attestation-20261001`. Path B is the route that can prove the meter and it is not built. Replication-write completeness is a launch blocker: it is not waived and not scoped out.** See "Round six: why path A is parked" at the end. |
| ## Where the meter stands |
| @@ -56,6 +56,23 @@ Not built, in the order it should be: |
| What an excluded account sees: its replication-write meter never completes, so the monthly freeze refuses its month and it is billed nothing. Whether such an account should instead be billed its other four meters with writes left out is a product decision this plan does not make. |
| +## Round six: why path A is parked (2026-10-01) |
| + |
| +Both reviewers read path A end to end at `058ea506`. Astra R7 to R10, R12, R14 and Opus N1, N3 to N9 are open against it (`docs/superpowers/specs/CODEX-FINDINGS-carloku-paid-launch-overnight-round6.md`, `OPUS-REVIEW-carloku-paid-launch-overnight-round6.md`). Two of them are not repairable inside path A's design: |
| + |
| +- **A config's modification time is not the running configuration** (Astra R9, Opus N3, N8). The reader infers what the replicator has open from a file and a unit's start time. A replicator that was restarted, or a file touched without being changed, breaks the inference in either direction, and a replicator started and stopped between two ticks is never seen at all (R8). |
| +- **One named unit is not the inventory of writers** (Astra R10, Opus N5). The attestation says "every write to the replica bucket for these accounts went through the ledger". Inspecting `carlos-litestream.service` shows what that unit does. It shows nothing about a second replicator, a hand-run binary, or anything else on the box holding the bucket's credentials. |
| + |
| +An attestation that excludes the owners it knows about and vouches for everyone else is only as good as its knowledge of who else can write. Path A has no way to know that. So it is not a smaller version of the proof; it is not a proof. |
| + |
| +The rest are real and fixable, and would matter if path A came back: a new exclusion dated back over days already published complete (Opus N1), shutdown sealing without refreshing the scope (R7), symlinks resolved at tick time (N4), a new box's first day and every restart day holding for everyone (R12, N6), unit tenants' paths unresolvable by the agent (N7), `$` accepted in generated paths (N9), and a padded switch value accepted (R14). |
| + |
| +What stays on the launch branch from this work: the console's per-account durability days (`DurabilityExcluded`, `accountSourceDays`), which any attestation with an exclusion set needs, and the fold's rule that a box proves the union of its windows with no tolerance. Nothing on the branch produces an attestation. |
| + |
| +**What can prove the meter: path B.** Make the platform's own replicator the only process with the replica bucket's credentials, and have it count what it writes. Then "every write went through the ledger" is true by construction (credentials) and not by inspection. Concretely: `carlos ops replicate` running the vendored Litestream with the request-meter middleware, replacing the external binary in `carlos-litestream.service`; the external binary's credentials removed from the box; a startup check that refuses to attest while the external unit exists. That is a root unit change on every box, so it needs a scoped production approval, a dev box first, and the flagship's hand-kept config cut over. None of it is built. |
| + |
| +Until then: `durability_writes_1k` is incomplete for every month on every box, the monthly freeze is refused, and no overage invoice is issued. That is an unproven meter holding, as designed. It blocks the launch of the complete metered model. |
| + |
| ## Open decisions |
| - Whether LIST requests, which S3 prices at the write tier, should count. They do not today. |
diff --git a/docs/superpowers/specs/2026-10-01-stripe-idempotency-evidence.md b/docs/superpowers/specs/2026-10-01-stripe-idempotency-evidence.md| new file mode 100644 |
| index 000000000..429a8f07a |
| --- /dev/null |
| +++ b/docs/superpowers/specs/2026-10-01-stripe-idempotency-evidence.md |
| @@ -0,0 +1,113 @@ |
| +# Stripe idempotency: what the quantity fence rests on, and what is proven |
| + |
| +Date: 2026-10-01. Status: **documentation evidence only. Not verified against Stripe.** The team quantity worker stays off until the sandbox check below has passed. |
| + |
| +## Why this exists |
| + |
| +The team quantity worker (`internal/console/billingquantity.go`) retires an operation that did not run and replaces it with a newer one. A request for the retired operation may still be on its way: its sender was paused after its last check, or the request was held in transit. A timed lease in the console's database cannot stop that request. Only the provider can. |
| + |
| +The fence: before an operation nobody can show ran is acknowledged, its idempotency key is consumed at Stripe with a metadata-only update of the same subscription item (`billingstripe.FenceQuantityKey`). The claim is that Stripe then refuses, or answers from memory, any later quantity update under that key. |
| + |
| +That claim is about Stripe's behaviour. The tests in `internal/console` use a fake provider written from the rules quoted below. They prove the console's logic given those rules. They do not prove the rules. |
| + |
| +## What Stripe documents |
| + |
| +Retrieved 2026-10-01. Quotes are verbatim. |
| + |
| +Source A: <https://docs.stripe.com/api/idempotent_requests> |
| +Source B: <https://docs.stripe.com/error-low-level> |
| +Source C: <https://docs.stripe.com/api/errors> |
| +Source D: <https://docs.stripe.com/error-codes> |
| + |
| +### Which responses persist a result for a key |
| + |
| +> Stripe's idempotency works by saving the resulting status code and body of the first request made for any given idempotency key, regardless of whether it succeeds or fails. Subsequent requests with the same key return the same result, including `500` errors. (A) |
| + |
| +> We save results only after the execution of an endpoint begins. If incoming parameters fail validation, or the request conflicts with another request that's executing concurrently, we don't save the idempotent result because no API endpoint initiates the execution. You can retry these requests. (A) |
| + |
| +> For a `POST` operation using an idempotency key, as long as an API method began execution, Stripe's API servers will cache the results of the request regardless of what they were. A request that returns a `400` sends back the same `400` if followed by a new request with the same idempotency key. [...] This operation does contain some caveats. For example, a request that's rate limited with a `429` can produce a different result with the same idempotency key because rate limiters run before the API's idempotency layer. The same goes for a `401` that omitted an API key, or most `400`s that sent invalid parameters. (B) |
| + |
| +> As with user errors, the idempotency layer caches the result of `POST` mutations that result in server errors (specifically `500`s, which are internal server errors) [...] You should treat the result of a `500` request as indeterminate. (B) |
| + |
| +Consequences for the design: |
| + |
| +- A fence request answered 2xx has taken the key. |
| +- A fence request answered 429, 401, or a validation 400 has taken nothing. The code treats every such answer as not fenced. |
| +- A fence request answered 500 is indeterminate. The code treats it as not fenced and asks again; if the key was in fact saved, the next ask replays the saved 500 and stays not fenced. An operation in that state is held, not acknowledged. That is safe and can stick for as long as Stripe remembers the key. |
| +- A quantity update refused for its `proration_date` is a validation failure and saves nothing, so the worker fences the key itself afterwards. Which 400s count as "most" is not specified. Sandbox check 3 covers this one. |
| +- A quantity update answered 500 may still have run, and Stripe says it may reconcile such requests later. No key rule covers that. The console sees the result only when it next reads the subscription. |
| + |
| +### Request-parameter comparison |
| + |
| +> The idempotency layer compares incoming parameters to those of the original request and errors if they're not the same to prevent accidental misuse. (A) |
| + |
| +> Sending the same idempotency with different parameters produces an error indicating that the new request didn't match the original. (B) |
| + |
| +> `idempotency_error`: Idempotency errors occur when an `Idempotency-Key` is re-used on a request that does not match the first request's API endpoint and parameters. (C) |
| + |
| +The documentation names the error type. It does not state the HTTP status for a mismatch. The code requires status 400 with type `idempotency_error` before it counts a key as held. If Stripe answers a mismatch with another status, the code reads it as not fenced, which holds the operation and never acknowledges it wrongly. Sandbox checks 1 and 2 establish the real status. |
| + |
| +The comparison is against "the original request". If the original saved nothing (validation failure), there is nothing to compare with and the later request runs as new. That is why the fence is taken explicitly and not inferred from an earlier refusal. |
| + |
| +### Concurrent requests with the same key |
| + |
| +> the request conflicts with another request that's executing concurrently, we don't save the idempotent result (A) |
| + |
| +> `409` Conflict: The request conflicts with another request (perhaps due to using the same idempotent key). (B, C) |
| + |
| +> `idempotency_key_in_use`: The idempotency key provided is currently being used in another request. This occurs if your integration is making duplicate requests simultaneously. (D) |
| + |
| +A fence request that meets the operation's own request mid-execution is answered 409 and saves nothing. The code treats 409 as not fenced, with no error, and asks again on the next pass. The reverse case (the operation's request arrives while the fence request is executing) is the same rule seen from the other side: the operation's request is answered 409 and does not run. Its sender sees an error and the next pass decides from a fresh read. |
| + |
| +### Retention and expiry |
| + |
| +> You can remove keys from the system automatically after they're at least 24 hours old. We generate a new request if a key is reused after the original is pruned. (A) |
| + |
| +> Clients can safely retry requests that include an idempotency key as long as the second request occurs within 24 hours from when you first receive the key (keys expire out of the system after 24 hours). (B) |
| + |
| +**A consumed key is not a permanent fencing token.** The fence holds for 24 hours from the request that took the key. After that a request under the same key "generate[s] a new request": it runs. |
| + |
| +## A sender paused beyond retention |
| + |
| +The schedule the fence does not cover: |
| + |
| +1. A sender takes the send lease and is frozen before its request leaves it. |
| +2. The lease runs out. Another process fences the key, retires the operation, records a replacement for a higher quantity, and that lands. |
| +3. More than 24 hours after step 2 the sender thaws and sends. Stripe has forgotten the key. The request runs and lowers the quantity to the old target. |
| + |
| +What bounds it: |
| + |
| +- The sender's request carries a context deadline tied to its lease. A process stopped by a signal wakes to an expired deadline and sends nothing, because the clock the deadline runs on keeps counting. A suspended virtual machine is different: that clock stops with it. The deadline is therefore not a bound for a suspended host, and this design does not claim it is. |
| +- The request carries `proration_date`, which Stripe refuses outside the subscription's current period. On an annual plan that is a wide window. It stops a charge into a later period and little else. |
| +- Detection. The console never lowers a quantity inside a period, so a quantity observed to have gone down files `billing_quantity_lowered`, which holds the account's invoices until a person has looked (`TestQuantityLoweredAtTheProviderFilesAReview`). Before 2026-10-01 04:50 this did not exist: the console followed the provider's number down with no review. An earlier comment in the code said a mismatch review covered this case. It did not, and the comment was wrong. |
| + |
| +So past retention the outcome is **detected and held, not prevented**. Preventing it needs a provider-side precondition Stripe's subscription item update does not offer. Closing it fully means changing how seats are charged (for example a subscription schedule phase per change, or invoice items in place of quantity changes). That is not built and not proposed for this launch. |
| + |
| +The same limit applies to the invoice engine (`internal/console/billinginvoice.go`). Its operations are never replaced, so within retention a late request is an identical replay. Past retention a late request runs as new: a second empty draft, a duplicate line that `validateDraftLines` catches before finalize, or, in one narrow schedule, a finalized invoice with a wrong total that `verifyFinalized` flags and the engine then refuses to charge. Whether the customer could pay such an invoice from Stripe's hosted page was not determined. |
| + |
| +## The gate |
| + |
| +The worker runs only when both are set: |
| + |
| +- `CONSOLE_BILLING_PRORATION` (the commercial decision, Paul's), and |
| +- `CONSOLE_BILLING_QUANTITY_FENCE=stripe-sandbox-verified` (the engineering proof). |
| + |
| +The second is set only after `TestSandboxQuantityFence` (`internal/billingstripe/sandbox_fence_test.go`) has passed against a Stripe test-mode account and its log is kept with the launch evidence: |
| + |
| +``` |
| +CARLOS_STRIPE_SANDBOX_KEY=<rk_test_ key, from the environment, never on a command line that is logged> \ |
| + go test ./internal/billingstripe/ -run TestSandboxQuantityFence -count=1 -v |
| +``` |
| + |
| +It needs a test-mode restricted key that can write customers, products, prices and subscriptions and read invoices. It refuses any key that is not `rk_test_`. It has **not been run**: test mode is not authorized for this overnight run. |
| + |
| +| Check | What it establishes | Status | |
| +| --- | --- | --- | |
| +| 1 | A fence request is accepted, changes neither the quantity nor any billing object, and a quantity update arriving after it is refused with 400 `idempotency_error` | not run | |
| +| 2 | A second copy of a quantity update is answered from memory and creates no second proration; a fence under that key is refused with 400 `idempotency_error` | not run | |
| +| 3 | A `proration_date` outside the period is refused naming the parameter and saves no key | not run | |
| +| 4 | Concurrent copies of one request run it once; the 409 answer if it is observed | not run | |
| +| 5 | A canceled subscription refuses a quantity update; what a fence on its item answers | not run | |
| +| none | Behaviour after the key is forgotten | cannot be checked in a run; documented as "runs as new" and treated that way | |
| + |
| +If check 5 shows Stripe refusing the fence on a canceled subscription's item, the worker's ended-subscription path would file `billing_quantity_fence_refused` for every canceled team subscription with an unconfirmed operation. That is safe and noisy, and needs a code decision before the worker is switched on. |
diff --git a/docs/superpowers/specs/CODEX-DISPOSITIONS-carloku-paid-launch-overnight.md b/docs/superpowers/specs/CODEX-DISPOSITIONS-carloku-paid-launch-overnight.md| index 97e0f487e..b1db452cd 100644 |
| --- a/docs/superpowers/specs/CODEX-DISPOSITIONS-carloku-paid-launch-overnight.md |
| +++ b/docs/superpowers/specs/CODEX-DISPOSITIONS-carloku-paid-launch-overnight.md |
| @@ -164,3 +164,62 @@ Not changed, with reasons: |
| - A teardown that leaves a bucket to another account keeps its record and returns an error until the other store is torn down or a person removes the bucket. There is no operator override yet. |
| - The scanner debt is only carried for apps that leave through the current code. A plan enabled by an app torn down by an older build, on a bucket still shared, is not known to the ledger. |
| - Stripe's behaviour on a proration date, and whether its refusal names the parameter, is still unconfirmed. If it does not name it, a refused operation is not retired at once and falls back to the replay limit. |
| + |
| +## Round six (reviewed `058ea506`; Astra rejected, Opus accepted with fixes) |
| + |
| +Reports: `CODEX-FINDINGS-carloku-paid-launch-overnight-round6.md`, `OPUS-REVIEW-carloku-paid-launch-overnight-round6.md`. |
| + |
| +Six rounds each found a new interleaving of the same two faults: authority over a bucket name bound to an app and not to an invocation, and sending a quantity operation racing its retirement. Supervision's direction was to stop fixing examples. This round's fixes are two mechanisms, each stated as an invariant, each with a randomized simulation that checks the invariant on every schedule it generates, and each with a mutation pass that removes the mechanism and confirms a test fails. |
| + |
| +A timed lease does not stop a request that has already left its sender. Both mechanisms therefore end at the backend: AWS refuses a walk's late request by its signing time, and Stripe refuses a retired operation's late request by its idempotency key. The first is verified against AWS. The second is **not verified against Stripe** and is bounded in time. Both statements are spelled out below. |
| + |
| +| Finding | Disposition | Where | Reproduction | |
| +| --- | --- | --- | --- | |
| +| Astra R1: duplicate teardowns of one app can destroy a newly granted bucket | Accepted, at the mechanism. One walk at a time per bucket name, across processes: a lease object taken by compare-and-swap and bound to the invocation. A walk that ends in error keeps the name reserved until its lease runs out. | `objstore/walklock.go` | `TestWalkLeaseIsOneInvocationAtATime`, `TestWalkLeaseStandsAfterAWalkThatFailed`, the simulation | |
| +| Astra R1, supervision 04:00: a paused holder acts after its lease is taken | Accepted. Every request of a walk is signed with the instant its lease was taken. AWS refuses a request stamped more than fifteen minutes from its clock, and the lease lasts twenty, so a lease cannot be taken over while any request of the walk that held it could still be accepted. | `objstore/fence.go`, `aws.go` | `TestPausedWalkCannotActAfterItsLeaseIsTaken`, `TestFencedSignerStampsTheLeaseInstant`; read-only against AWS: `TestFenceAgainstAWS` (S3 and IAM accept a stamp 13 minutes old and refuse one 16 minutes old; run 2026-10-01 against the flagship account, list and get calls only) | |
| +| Astra R2: a stale teardown marks a recreated bucket as already deleted | Accepted. The note is written with the record's carried etag inside the lease. A grant over an unfinished teardown voids the note before it does anything that could bring the bucket back. | `walk.go` | `TestTeardownAfterAFailedTakeBackDeletesTheBucketTheGrantRecreated` (found by the simulation at seed 1085 after this round's first fixes) | |
| +| Astra R3: a same-account holder hides another account's retained access | Accepted. `foreign` is any open interval or record of another account, leaving or not, read again from the ledger before the teardown finishes. | `ownership.go`, `walk.go` | the simulation's Quiet invariant (fails at seed 5 with the check removed) | |
| +| Astra R4: exclusive minting ignores a legacy adopter | Accepted. A minting grant reads every store record and is refused while another names the bucket. | `walk.go` `otherStoreHolders` | `TestRetriedLegacyTeardownLeavesABucketAnotherAppNowHolds` | |
| +| Astra R11: recovery clears its rebuild requirement before rebuilding | Accepted. The teardown block is a write-ahead record cleared only by the write that records the rebuilt store. A refused grant undoes the grant block it wrote. | `walk.go` | `TestGrantOverItsOwnUnfinishedTeardownTakesTheStoreBack`, the simulation | |
| +| Opus N2: after a retained teardown a re-grant reports a dead key healthy, and the teardown can never finish | Accepted. The teardown block makes the re-grant run in full. The destroyer stamps `DestroyedAt`, and a retained teardown whose store left before that finishes. Its member copy is not written: it needs Paul's review. | `walk.go`, `ownership.go` | `TestRetainedTeardownFinishesOnceTheBucketItLeftIsDestroyed` | |
| +| Astra R5, Opus "other answers": retirement can overtake a sender after its final re-read | Accepted, at the mechanism. A send lease: no request is issued outside it and no operation is retired or replaced under it. | `billingquantity.go` `claimQuantitySend` | `TestQuantityOperationIsNotRetiredAcrossTheReplayLimitUnderItsSender`, `TestQuantityNothingIsConcludedWhileASenderHoldsTheLease` | |
| +| Supervision 04:00: a request issued inside the lease and arriving after it | Accepted, **bounded, unverified**. Before an operation nobody can show ran is acknowledged, its key is consumed at the provider with a metadata-only request; a retirement is decided on a provider read taken after that; a key is never reused. See "What the quantity fence does not prove". | `billingstripe.FenceQuantityKey`, `billingquantity.go` | `TestQuantityPausedSenderIsRefusedByTheProviderAfterItsOperationIsRetired`, `TestQuantityRequestThatRanJustBeforeTheFenceIsRecordedNotRetired`, `TestQuantityOperationIsNeverAcknowledgedUnfenced`, the simulation's Fenced invariant. All against a fake provider. | |
| +| Astra R6: intervening quantity revisions bypass the unconfirmed-operation hold | Accepted. The hold is filed in the transaction that clears the addition, not by whichever reconcile saw the change. | `billingquantity.go` | `TestQuantityUnverifiedAdditionIsNeverClearedWithoutAHold` | |
| +| Astra R13: a successful retry loses its confirmation after a provisional acknowledgment | Accepted. Nothing is concluded about an operation while a sender holds its lease, by the worker or by a webhook reconcile. | `billingquantity.go`, `billingservice.go` | `TestQuantityReconcileWaitsWhileASenderHoldsTheLease` | |
| +| Astra R7, R8, R9, R10, R12, R14; Opus N1, N3 to N9: scoped attestation | Accepted as findings. **Not fixed.** R9 and R10 show path A cannot prove an exclusive writer: a modification time is not the running configuration, and one named unit is not the inventory of writers. The scope, the replicator reader and the agent switch are removed from this snapshot and kept on `feat/carloku-scoped-attestation-20261001` (at `058ea506`) with these findings open against it. | removed from `internal/host`, `internal/ops` | none | |
| +| Astra section 3: a statement unreadable for three passes | Accepted as a wording fault. The three-pass rule applies to an account's statement. The platform's stays unknown. | handoff | none | |
| + |
| +### Replication-write completeness is a launch blocker |
| + |
| +Removing path A does not waive the meter. `durability_writes_1k` is billed in the approved model, production attests nothing, and so every month's meter is incomplete and no overage invoice can be issued. That is the correct behaviour of an unproven meter and it blocks the launch. The route that can prove it is path B in `2026-10-01-durability-attestation-path.md`: the platform's own replicator as the only writer, counting what it writes. It is not built. |
| + |
| +### What the quantity fence does not prove |
| + |
| +`docs/superpowers/specs/2026-10-01-stripe-idempotency-evidence.md` quotes Stripe's documentation and maps each rule to the code. |
| + |
| +- The tests run against a fake provider that models the documented rules. They prove the console's logic given the rules. They do not prove the rules. Nothing has been sent to Stripe: test mode is not authorized. |
| +- A consumed key is not a permanent fencing token. Stripe documents that keys may be removed after 24 hours and that a key reused after that runs as a new request. A sender frozen for more than a day can still land its request. That case is detected and held (`billing_quantity_lowered`, added this round), not prevented. |
| +- The documentation does not state the HTTP status of a parameter mismatch. The code requires 400 with type `idempotency_error` and treats anything else as not fenced. |
| +- The worker has two switches and both are unset: `CONSOLE_BILLING_PRORATION` and `CONSOLE_BILLING_QUANTITY_FENCE=stripe-sandbox-verified`. The second is set only after `TestSandboxQuantityFence` passes against a Stripe test-mode account. It has not been run. |
| + |
| +Found while checking this: the console followed the provider's quantity down inside a period with no review. An earlier comment claimed a mismatch review caught it. That was wrong. It now files `billing_quantity_lowered`. |
| + |
| +### What the simulations found after the first fixes |
| + |
| +- Storage, seed 778 (beyond the default 400): the simulation's own bookkeeping. A teardown refused as busy did nothing, and the model forgot the app's reported retained state. Fixed in the model. |
| +- Storage, seed 1085: a real fault, R2's sibling, fixed as above. 30,000 seeds pass after it. |
| +- Quantity: 500 seeds pass with a request held past its lease, once per run for 23 hours. |
| + |
| +### Mutation checks (run in a scratch copy of the tree) |
| + |
| +Each removes one piece of a mechanism. All fail a test except one equivalent mutant. |
| + |
| +- Quantity: fence without asking the provider; store rule dropped; retire on the read before the fence; a fenced operation claimable; fence under a live lease; unverified recorded without the fence; a fenced operation falling through to send. |
| +- Storage: signer not pinning the lease instant; backend never refusing an old stamp; walk calls carrying no fence; a live lease taken over; a failed walk freeing the name; lease shorter than the backend window; destructive step not re-confirmed. |
| +- Equivalent: dropping the leaving-holder count in `beginBucketTeardown` alone, because the teardown reads the ledger again before finishing. Dropping both fails the simulation at seed 5. |
| + |
| +### Still open from this round |
| + |
| +- The invoice engine's lease has no provider-side fence beyond identical replays. Its operations are never replaced, so within Stripe's key memory a late request is a replay. Past it a late request runs as new and is detected and held, not prevented. One schedule leaves an open invoice with a wrong total that the engine refuses to charge; whether the customer could pay it from Stripe's hosted page is not determined. |
| +- `ErrBucketRetained` has no member copy. It needs Paul's review. |
| +- A fence request answered 500 leaves its operation outstanding for as long as Stripe remembers the key. |
diff --git a/docs/superpowers/specs/CODEX-FINDINGS-carloku-paid-launch-overnight-round6.md b/docs/superpowers/specs/CODEX-FINDINGS-carloku-paid-launch-overnight-round6.md| new file mode 100644 |
| index 000000000..91786b0ae |
| --- /dev/null |
| +++ b/docs/superpowers/specs/CODEX-FINDINGS-carloku-paid-launch-overnight-round6.md |
| @@ -0,0 +1,227 @@ |
| +## 1. Round-five dispositions |
| + |
| +Reviewed `058ea506` against `aed42815`, including `daa32cb6..058ea506`. All references identify the pinned commit. This was static review: no files changed, tests executed, or network commands run. Counterfactual test assessments below are based on reading the tests and implementation, not mutation runs. |
| + |
| +Path abbreviations: **C** = `internal/console`, **O** = `internal/objstore`, **H** = `internal/host`. |
| + |
| +| Finding | Disposition | Fix and reproducing test | Would the test pass without the fix? | |
| +|---|---|---|---| |
| +| Astra F1 | **Partly closed.** Single teardown versus another app’s grant and legacy-holder discovery are protected. Duplicate teardowns can release each other’s protection; finding R1. | `O/ownership.go:336`, `:430`, `:485`; `O/walk.go:598`, `:782`. Tests: `O/ownership_test.go:769`, `:1048`, `:1124`. | **No** with the corresponding protection removed. The concurrency test makes sequential ledger decisions; it never pauses a destructive backend call. | |
| +| Astra F2 | **Partly closed.** Future-dated operations are preserved and senders reread their operation. Retirement after that reread remains unsafe; R5. | `C/billingquantity.go:700`, `:795`. Tests: `C/billingquantity_test.go:1205`, `:1265`; assertions `:1239`, `:1278`. | **No** without the corresponding guard. Both schedules pause before the reread. | |
| +| Astra F3 | **Closed for the reported two-app departure and scanner-debt cases.** | Leaving decision and debt: `O/ownership.go:449`, `:470`, `:473`, `:481`; consumption: `O/walk.go:632`. Tests: `O/ownership_test.go:1082`, `:993`, `:905`. | **No** without Leaving/debt handling. The “concurrent teardowns” test checks sequential decisions, not concurrent complete walks. Scanner assertions now cover removal and refusal when no scanner is supplied. | |
| +| Astra F4 | **Closed for delay inside Grant.** | Elapsed time starts at `O/walk.go:148` and is sampled after the probes at `:233`. Test: `O/ownership_test.go:1167`. | **No** if the earlier caller timestamp is restored. The test substitutes twelve seconds of elapsed time; it does not delay a probe. Source inspection establishes the ordering. | |
| +| Astra F5 | **Closed for the registration-window defect.** Healthy producer startup still holds; R12. | Window union: `C/durabilityfold.go:81`, `:96`, `:100`, `:141`. Test: `C/meterfolds_test.go:562`; assertions `:592`, `:603`, `:614`, `:620`. | **No** with the old comparison/tolerance restored. The fixture supplies registration and attested coverage directly; it does not exercise production startup. No case places an exclusion solely inside the previously omitted minutes. | |
| +| Astra F6 | **Closed for nested, duplicate and symlink paths.** | `H/durabilityscope.go:129`, `:151`; `H/durabilityreplicator.go:124`. Tests: `H/durabilityscope_test.go:58`, assertion `:107`; `H/durabilityreplicator_test.go:173`. | **No** with first-match attribution or the corresponding path resolution removed. | |
| +| Astra F7 | **Closed for an active unit with absent configuration.** | `H/durabilityscope.go:107`; `H/durabilityreplicator.go:97`. Tests: `H/durabilityscope_test.go:56`; `H/durabilityreplicator_test.go:101`. | **No** when the guard at the corresponding tested layer is removed. | |
| +| Astra F8 | **Closed.** Absent-month memory expires. | `C/economicspass.go:1352`, `:1404`, `:1486`. Test: `C/meterfolds_test.go:926`, expiry and assertion `:948`. | **No** without expiration. The fixture ages the cache timestamp instead of waiting six hours. | |
| +| Astra F9 | **Partly closed.** Direct coincidental matching is held; intervening quantity revisions bypass that hold, R6. | `C/billingquantity.go:727`, `:787`; `C/billingservice.go:500`; `C/billingstore.go:454`, `:528`. Test: `C/billingquantity_test.go:1032`; assertions `:1045`, `:1049`, `:1058`. | **No** for the original backdating, missing-review and incorrect-outcome regressions. | |
| +| Astra F10 | **Closed in code; reproduction insufficient.** | Transactional replacement: `C/billingquantityaudit.go:38`. Test: `C/billingquantity_test.go:1473`. | **Yes.** The opens at `:1499` and `:1508` are sequential. Separately committed DROP/CREATE also passes. | |
| +| Astra F11 | **Closed in documentation.** | `docs/superpowers/plans/2026-10-01-partial-month-billing.md:17`, `:18`, `:32`; `C/billingcalculation.go:143`. Existing behavior tests: `C/billingcalculation_test.go:218`, `:292`. | **Yes.** This corrects prose, not behavior; those tests cannot detect reverting the prose. | |
| +| Opus N1 | **Partly closed.** Two-holder retention and ledger-backed exclusive minting are repaired. Mixed holders and legacy minting remain unsafe; R3–R4. | `O/ownership.go:355`; `O/walk.go:718`. Tests: `O/ownership_test.go:769`, `:870`, `:905`. | **No** without the corresponding guards. No fixture covers three mixed-account holders or a legacy adopter during exclusive minting. | |
| +| Opus N2 | **Closed for the reported existing legacy record.** | `O/walk.go:782`; `O/ownership.go:454`. Test: `O/ownership_test.go:1048`. | **No** without record inventory. | |
| +| Opus N3 | **Closed for current-code departures.** Older-build orphan plans remain outside recovery. | `O/ownership.go:473`; `O/walk.go:632`. Tests: `O/ownership_test.go:993`, `:905`. | **No** without propagation or consumption of scanner debt. | |
| +| Opus N4 | **Partly closed.** The marker protects one teardown against another app’s grant, but identifies an app rather than a teardown invocation; R1–R2. | `O/ownership.go:336`, `:485`, `:497`; `O/walk.go:661`, `:670`. Tests: `O/ownership_test.go:769`, `:1124`. | **No** without marker checks. Neither test covers duplicate teardowns or a same-app regrant during destruction. | |
| +| Opus N5 | **Closed.** | Addition-time ceiling: `C/billingquantity.go:150`, `:639`; transactional check `:409`. Tests: `C/billingquantity_test.go:1286`; compare-and-set test `:677`, case `:703`. | **No** without the relevant bounds. However, assertion `:1299` compares against `added.Unix()` and does not independently pin upward rounding of fractional seconds. | |
| +| Opus N6 | **Closed for non-platform statements, as requested.** Platform read failures remain Unknown. | `C/economicspass.go:1430`. Test: `C/meterfolds_test.go:850`; platform assertions `:886`, tenant threshold `:904`. | **No** without the threshold or Unknown behavior. | |
| +| Opus N7 | **Partly closed.** Path attribution, command checks and changes during a probe are covered. Scope continuity and final sealing remain unsafe; R7–R10. | `H/durabilityreplicator.go:87`, `:124`, `:153`, `:167`; `H/durabilityscope.go:259`. Tests: `H/durabilityreplicator_test.go:110`, `:124`, `:173`; retrospective test `H/durabilityscope_test.go:318`. | **No** for the corresponding command/path/coherent-read checks. **Yes** if either retrospective dating bound is removed individually: both fixture timestamps precede midnight, so either remaining bound holds the day. | |
| +| Opus N8 | **Partly closed.** The original direct coincidence now has the correct label and hold. The broader dating guarantee fails after intervening revisions, R6. | Same fix/test as Astra F9. Real successful-operation fixture: `C/billingquantity_test.go:996`; lost-response and young-operation cases `:295`, `:1064`. | **No** for the corresponding confirmation, outcome and wait regressions. The success fixture’s callback runs after confirmation and misses R13. | |
| +| Opus N9 | **Closed in code and documentation; reproduction insufficient.** | `C/billingquantityaudit.go:38`; rollback warning in `implementation/carloku-paid-launch/AGENT_HANDOFF.md:63`. Test: `C/billingquantity_test.go:1473`. | **Yes** without transactional replacement. No rollback reproduction is provided. | |
| + |
| +## 2. New findings |
| + |
| +### R1. P1 — Duplicate teardowns can destroy a newly granted bucket |
| + |
| +**Where:** `internal/objstore/ownership.go:485`, `:502`, `:398`; `internal/objstore/walk.go:661`, `:667`. |
| + |
| +**Scenario:** Two console processes start teardown T1 and T2 for the same app. Both receive destructive authority because `Destroying` identifies their app, not their invocation. T2 passes its confirmation immediately before `EmptyBucket` and pauses. T1 finishes, clears the marker and removes the record. Another app adopts the freed name and writes data. T2 resumes and empties that new bucket. |
| + |
| +The subsequent confirmation refuses, but the data has already been deleted. The console lock is explicitly process-local (`internal/console/storelock.go:27`). A same-app Grant can also clear the marker at `ownership.go:340`. |
| + |
| +**Smallest acceptable fix:** Serialize complete destructive walks and regrants across processes. One invocation must not release another invocation’s authority while its backend calls can still execute. Add a reproduction paused after successful confirmation and before the actual destructive call; another reread alone is insufficient. |
| + |
| +### R2. P1 — A stale teardown marks a recreated bucket as already deleted |
| + |
| +**Where:** `internal/objstore/walk.go:678`, `:683`, `:595`; `internal/objstore/ownership.go:340`. |
| + |
| +**Scenario:** An earlier Grant crashed after creating its estate but before recording `Provisioned`. Teardown deletes that bucket, then pauses before rereading the record. Another console reruns Grant for the same app: it clears the marker, recreates the bucket and credentials, and records the new estate. |
| + |
| +The old teardown now reads the **new** record and successfully attaches `Teardown.BucketDeletedAt` using its freshly read ETag. It deletes the recreated user. The final key guard rejects removal of the changed record, leaving the false deleted-bucket marker. |
| + |
| +A subsequent teardown sees `bucketDone=true`, skips the live bucket and removes the record. The bucket is orphaned. |
| + |
| +**Smallest acceptable fix:** Bind the deleted-bucket marker to the exact estate generation actually deleted, checking the original generation before writing it. Also exclude overlapping walks as in R1. A fresh ETag alone does not establish that the fresh record describes the deleted estate. |
| + |
| +### R3. P1 — A same-account holder hides another account’s retained access |
| + |
| +**Where:** `internal/objstore/ownership.go:449`, `:453`; `internal/objstore/walk.go:718`. |
| + |
| +**Scenario:** A bucket has holders, in ledger order, `bab/one`, `bab/two`, `bac/three`. Tearing down `bab/one` selects `bab/two` as its single “other” holder. It revokes its credentials, sees the selected holder belongs to the same account, deletes its record and reports success. Its objects remain readable by `bac/three`. |
| + |
| +The same failure occurs when a same-account ledger holder masks a different-account legacy holder: record inventory is considered only if no ledger holder was selected. |
| + |
| +**Smallest acceptable fix:** Calculate whether **any** surviving holder belongs to another account across all ledger and applicable inventory entries. Use that result for `ErrBucketRetained`; select one holder only for diagnostics. Test both ledger orders and mixed ledger/legacy holders. |
| + |
| +### R4. P1 — Exclusive minting ignores a legacy adopter |
| + |
| +**Where:** `internal/objstore/ownership.go:343`, `:355`; `internal/objstore/walk.go:230`, `:240`, `:265`. |
| + |
| +**Scenario:** A legacy store in account B has adopted the deterministic bucket name that account A’s new store would mint. B has a store record and working credentials but no ownership interval. |
| + |
| +A’s Grant finds the existing bucket and grantees. Those observations merely prevent a completeness claim; they do not refuse access. The exclusive check considers only ledger intervals, so it succeeds and delivers A credentials for B’s existing data. |
| + |
| +Teardown’s newly added record inventory does not protect this grant path. |
| + |
| +**Smallest acceptable fix:** Include existing store-record holders in exclusive-mint authorization and refuse unresolved existing ownership before creating credentials. Add an exclusive-mint test with a live legacy adopter absent from the ledger. |
| + |
| +### R5. P1 — Retirement can overtake a sender after its final reread |
| + |
| +**Where:** `internal/console/billingquantity.go:700`, `:705`, `:803`. |
| + |
| +**Scenario:** |
| + |
| +1. An outstanding 10→11 operation has never executed. |
| +2. Recovery worker A reads it at age 21:59:59, obtains provider quantity 10, passes the replay-age check and final outstanding-operation reread, then pauses. |
| +3. Worker B reads at age 22 hours, sees quantity 10 and retires the operation. |
| +4. Another member has joined. Worker C completes a replacement 10→12 operation. |
| +5. A resumes seconds later, within its fresh send window, and submits its previously unused key. The old request lowers quantity from 12 to 11. |
| + |
| +The comment that no sender can remain inside its send window is false: recovery obtains a fresh provider-read timestamp. |
| + |
| +**Smallest acceptable fix:** Coordinate send authorization and retirement across processes so replacement cannot become eligible while an old sender can still execute. Add a test paused **after** the final reread and crossing the replay cutoff. |
| + |
| +### R6. P1 — Intervening quantity revisions bypass the unconfirmed-operation hold |
| + |
| +**Where:** `internal/console/billingservice.go:495`, `:500`; `internal/console/billingquantity.go:777`, `:547`; `internal/console/billingstore.go:450`. |
| + |
| +**Scenario:** An unconfirmed 10→11 operation never executes. Before the next worker pass, an operator independently sets provider quantity to 12, then to 11. |
| + |
| +The first webhook reconciles 10→12; the operation’s target does not match, so no dating review is filed. The second reconciles 12→11; its target matches, but `op.FromQuantity == state.Quantity` fails because stored quantity is now 12. Again, no review is filed. |
| + |
| +The worker subsequently observes 11 and returns `landed_unverified`. Reconciliation sees no remaining quantity change. The worker processes the pending addition and acknowledges the operation without filing a dating hold. |
| + |
| +Allowance history now uses observation times without proof of when the independent charges began, and the operation no longer prevents freezing. |
| + |
| +**Smallest acceptable fix:** Persist the dating hold before clearing an unconfirmed operation’s obligations, independently of whether reconciliation still sees its original from→target transition. Test the two-webhook schedule. |
| + |
| +### R7. P1 — Shutdown seals and publishes without refreshing attestation |
| + |
| +**Where:** `internal/host/durability.go:199`, `:211`; `internal/usage/durabilityledger.go:2063`, `:2082`. |
| + |
| +**Scenario:** An established ledger verifies its scope at 23:56. Account `bac` is hibernating and attested. At 23:58 its route becomes always-on, its database enters the generated external configuration, and the external unit restarts. The agent stops at 00:00:30, before another publish tick. |
| + |
| +`stop` calls `ledger.Close` directly. Close seals and publishes with the previous scope, allowing the preceding day to read complete for `bac` despite unmetered replication. |
| + |
| +Reopening does not repair that day: recovery’s gap begins at the final seal after midnight (`durabilityledger.go:795`), and the new run’s attestation begins at its new observation. |
| + |
| +**Smallest acceptable fix:** Refresh and apply scope under the publisher semaphore before the final seal. If the scope cannot be established, withdraw the claim from the last verified instant. Add an enabled-scope shutdown test with a change after the last tick. |
| + |
| +### R8. P1 — Replication that starts and stops between ticks disappears from coverage |
| + |
| +**Where:** `internal/host/durabilityscope.go:225`, `:256`; `internal/host/durabilityreplicator.go:153`. |
| + |
| +**Scenario:** At 23:56, `bac` is hibernating and absent from external replication. At 23:58 it becomes always-on, configuration is regenerated and the external unit starts replicating it. At 00:00 the route and configuration return to their earlier state and the unit restarts again. At 00:01 the scope reader observes a coherent, currently safe state. |
| + |
| +The unit/configuration checks compare observations within the current probe, not against the preceding tick. `applyScope` finds no newly excluded account and returns unchanged. The preceding day is attested despite the missing interval. |
| + |
| +This requires no false timestamp or torn read. |
| + |
| +**Smallest acceptable fix:** Preserve and compare configuration, unit and relevant ownership generations across observations. A changed generation must hold the intervening interval unless durable history establishes its scope. Add a change-and-revert-between-ticks test. |
| + |
| +### R9. P1 — Modification time is accepted as proof of the running configuration |
| + |
| +**Where:** `internal/host/durabilityscope.go:110`; `internal/host/durabilityreplicator.go:91`. |
| + |
| +**Scenario:** At 08:00 the external unit loads a generated configuration containing `bac`’s hibernating database. At 09:00 an older generated configuration omitting that database is restored while preserving its 07:00 modification time. The external process keeps running. The agent starts at 10:00 with scoped attestation enabled. |
| + |
| +Both reads are stable, and the unit start is later than the configuration mtime. The reader therefore vouches for `bac`, although the process still replicates it using the previously loaded configuration. Startup gaps may hold that first day, but subsequent full days remain incorrectly attested. |
| + |
| +**Smallest acceptable fix:** Require evidence identifying the configuration actually loaded by that replicator lifetime, such as a bound digest/generation. Hold when that identity cannot be established. Add a preserved-mtime replacement test. |
| + |
| +### R10. P1 — One named unit is treated as the complete external-writer inventory |
| + |
| +**Where:** `internal/host/durabilityreplicator.go:23`, `:44`, `:194`, `:221`; `internal/host/durabilityscope.go:164`. |
| + |
| +**Scenario:** Systemd reports `carlos-litestream.service` absent or inactive and the expected configuration is absent. Another unit, supervisor or manually launched Litestream is streaming `bac`’s hibernating database using another configuration. |
| + |
| +The probe accepts the absence of the named service as absence of external replication and attests `bac`. A second replicator alongside the expected unit is equally invisible. |
| + |
| +This is an accepted unsafe state in the code, not evidence that production currently has such a process. Failure to query systemd itself correctly withholds attestation. |
| + |
| +**Smallest acceptable fix:** Withhold scoped attestation unless the observed replication inventory is established as exclusive. An answer about one unit name cannot establish that property. |
| + |
| +### R11. P2 — Recovery clears its rebuild requirement before rebuilding |
| + |
| +**Where:** `internal/objstore/walk.go:162`, `:165`, `:276`. |
| + |
| +**Scenario:** Teardown deletes a provisioned store’s bucket, records `Teardown`, and fails at `DeleteUser`. The old `Provisioned` block and delivered variables remain. |
| + |
| +A recovery Grant immediately clears and persists `Teardown`. Its next `EnsureUser` fails, or the process crashes before `EnsureBucket`. The following Grant sees no teardown marker, sees the old provisioned record and delivered variables, and returns success through the healthy-store shortcut. Repeated grants continue reporting success while the bucket is absent. |
| + |
| +A related normal path needs no crash: `ErrBucketRetained` revokes the user but retains the provisioned record and variables without marking credentials as needing replacement. Regrant can recreate an empty-key user and return the old, dead credentials. |
| + |
| +**Smallest acceptable fix:** Persist “rebuild required” until the replacement estate and credentials have been recorded successfully. Retained teardowns must also record credential revocation. Extend `internal/objstore/ownership_test.go:832` with failure after the early update and a second Grant. |
| + |
| +### R12. P2 — A healthy new box’s first day cannot satisfy the fold |
| + |
| +**Where:** `internal/host/durability.go:54`, `:170`; `internal/host/durabilityscope.go:243`; `internal/console/durabilityfold.go:96`, `:141`; `internal/usage/durability.go:550`. |
| + |
| +**Scenario:** A fresh ledger opens at time `t0` without attestation. Its first successful scope observation occurs later, at `t1`, and attestation begins at `t1`. Source registration starts at `CreatedAt`, `t0`, or earlier. The fold requires attested coverage over that registered window, so `[t0,t1)` remains permanently unproven. |
| + |
| +This occurs on normal first starts with an advancing clock, even when the ledger opened before any traffic. It is an unnecessary first-day hold, not an incorrect charge. |
| + |
| +There is a second startup aggravation: `boxEarliestEvidence` chooses the earliest edge/metrics key boundary (`internal/host/usage_loop.go:289`). If another publisher has already written the startup day, registration can move back to midnight before the box existed. |
| + |
| +The fixed-clock scope fixtures make opening and initial observation simultaneous. The fold fixture supplies an already-attested ledger directly. |
| + |
| +**Smallest acceptable fix:** Establish a verified startup boundary before serving and preserve precise source-start evidence where available. Make registration and coverage agree with that boundary without waiving genuinely unproven time. Test real startup ordering with an advancing clock and different publisher orders. |
| + |
| +### R13. P2 — A successful retry can lose its confirmation after provisional acknowledgment |
| + |
| +**Where:** `internal/console/billingservice.go:514`; `internal/console/billingquantity.go:439`, `:727`. |
| + |
| +**Scenario:** Worker A retries a twenty-hour-old operation. The provider executes successfully, but A has not persisted the response. Worker B observes the resulting quantity. Because the operation’s creation time is older than the young-operation window, B does not wait: it records an unverified landing, files a review and acknowledges the operation. |
| + |
| +A then receives success. Confirmation updates only rows with `acked_at=0`, so it changes nothing and returns success. The valid response evidence is discarded; later passes cannot automatically establish the original date. |
| + |
| +This creates an unnecessary hold rather than an unheld wrong bill. The real-success fixture’s callback runs after confirmation (`internal/console/billingquantity_test.go:999`) and cannot reproduce it. |
| + |
| +**Smallest acceptable fix:** Preserve valid response evidence after provisional acknowledgment, and distinguish an active retry from the operation’s original creation time. |
| + |
| +### R14. P2 — Whitespace enables a switch that requires an exact value |
| + |
| +**Where:** `internal/host/durabilityscope.go:178`. |
| + |
| +**Scenario:** `CARLOS_DURABILITY_ATTEST=" scoped "` enables attestation because the value is trimmed. That violates the amended exact-value requirement. |
| + |
| +`internal/host/durabilityscope_test.go:189` explicitly expects the padded value to enable the switch. |
| + |
| +**Smallest acceptable fix:** Compare the environment value directly with `"scoped"` and make padded values negative cases. |
| + |
| +### Other requested boundaries |
| + |
| +- **Retained buckets:** An operator can finish by tearing down the other holder and retrying the retained store; `internal/objstore/ownership_test.go:971` demonstrates that route. Manually deleting the bucket alone does not clear the retained result while the other holder remains recorded. There is no selective override here. |
| +- **Scanner debt:** Current-code departures carrying `Scanning.Enabled` transfer debt, and the final destroyer consumes it or refuses without a scanner. Older-build orphan plans remain unrecovered. The stale estate-generation problem in R2 can still lose a live bucket’s cleanup record. |
| +- **Inventory and billing ownership:** `S3Store.ListDirs` paginates (`internal/release/store.go:746`). Record writes are atomic. The new operational fields do not change `OwnerAt`; Leaving intervals remain ownership ambiguities until actually closed. |
| +- **Quantity waiting:** A young-operation wait returns `errBillingPending`; webhook handling answers 503 rather than recording the event as consumed (`internal/console/billinghandlers.go:172`). I found no additional supported healthy-state scenario where the future-date rule waits forever. Actual provider treatment of future dates and period boundaries remains unverified. |
| +- **Statement memory:** I found no further demonstrated defect in the six-hour expiry or non-platform three-pass rule. Platform read failures intentionally remain Unknown. |
| +- **Attestation validation:** I found no additional repository path setting the switch on the box. A quiet week on a dev box cannot establish exclusive writers, loaded-configuration identity, between-tick history, or shutdown safety. Those require explicit adversarial coverage and stronger evidence. |
| + |
| +## 3. Handoff claims contradicted by the code |
| + |
| +All references below are to `implementation/carloku-paid-launch/AGENT_HANDOFF.md`. |
| + |
| +| Line | Claim | Contradiction | |
| +|---|---|---| |
| +| 33 | Scoped attestation is built end to end and enabled only by the specified environment value. | Shutdown bypasses scope verification; transient and unobserved external writers can remain attested; padded values also enable it. R7–R10, R14. | |
| +| 35 | The destroying marker refuses every grant until teardown finishes. | Same-app Grant explicitly clears it. Duplicate teardowns can also clear protection while another invocation remains destructive. R1–R2. | |
| +| 35 | A minting grant is refused while another app holds the name. | Exclusive minting examines ledger intervals but misses a legacy adopter’s record. R4. | |
| +| 35 | A teardown leaving a bucket to another account keeps its record and returns `ErrBucketRetained`. | Selecting a same-account holder first hides another-account holders and permits successful record deletion. R3. | |
| +| 36 | A statement unreadable for three passes is classified without it. | That applies only to non-platform statements. Platform read errors continue returning Unknown indefinitely. | |
| +| 38 | A sender does not send an operation that has been retired. | Retirement can occur after the final reread and before transmission. R5. | |
| +| 38 | An unconfirmed matching quantity is recorded with a review. | Intervening quantity revisions bypass review creation, allowing acknowledgment without a dating hold. R6. | |
| +| 62 | The first-day hold concerns a ledger opened after serving began. | That understates the affected population: ordinary fresh startup leaves an unattested interval even when the ledger precedes all traffic. R12. | |
| +| 78 | All previous findings were fixed with reproductions. | Several are only partly closed. The migration reproduction passes without transactional replacement, and several concurrency tests stop at sequential helper decisions. | |
| + |
| +## 4. Verdict |
| + |
| +**Reject.** Destructive-operation races, an unheld quantity-dating path, and false durability attestations remain in `058ea506`. |
| \ No newline at end of file |
diff --git a/docs/superpowers/specs/CODEX-REVIEW-BRIEF-carloku-paid-launch-overnight-round7.md b/docs/superpowers/specs/CODEX-REVIEW-BRIEF-carloku-paid-launch-overnight-round7.md| new file mode 100644 |
| index 000000000..89139310f |
| --- /dev/null |
| +++ b/docs/superpowers/specs/CODEX-REVIEW-BRIEF-carloku-paid-launch-overnight-round7.md |
| @@ -0,0 +1,33 @@ |
| +# Review brief, round 7: Carloku paid launch, overnight changes of 2026-10-01 |
| + |
| +Read the round 1 brief (`CODEX-REVIEW-BRIEF-carloku-paid-launch-overnight.md`) for the fixed rulings and the output rules, and the round 3 to 6 briefs for rulings 6 to 11. Then read `CODEX-DISPOSITIONS-carloku-paid-launch-overnight.md`, whose last section covers round 6, and `docs/superpowers/specs/2026-10-01-stripe-idempotency-evidence.md`. |
| + |
| +Review the commit named in the prompt. Read source only with `git show <commit>:<path>`: the working tree may be ahead of it. The changes since round 6 are `git diff 058ea506 <commit>`. |
| + |
| +This round is a **fixed snapshot**. Nothing was added to it beyond the round 6 fixes: no Free-tier work, no renewal work. Scoped attestation (path A) is removed from it, so ruling 10 now reads: nothing on this branch may attest. Any path that produces an attested ledger span is a finding. |
| + |
| +Ruling 12 (new). A timed lease does not fence a remote side effect. Where code relies on a lease to keep a stale holder from acting on a provider, it must be the provider that refuses the stale request, and the document must say exactly how far that holds. A claim of fencing that rests on a client-side check, a context deadline, or a fake provider's behaviour presented as the real one's is a finding. |
| + |
| +## First: a disposition table |
| + |
| +For every round 6 finding (Astra R1 to R14, Opus N1 to N9): closed, partly closed, not closed, or parked with the code removed. Give the file and line of the fix and of the test, and whether the test would pass without the fix. The dispositions list mutation checks; verify at least three of them by reading, and say which. |
| + |
| +## Then: new findings only |
| + |
| +The fixes this round are two mechanisms. Attack the mechanisms, not the examples. |
| + |
| +- **The walk lease and the backend fence** (`internal/objstore/walklock.go`, `fence.go`, `aws.go`, `memprovisioner.go`, `walk.go` `Grant` and `Teardown`). The invariant claimed: no provider call of a walk executes outside that walk's lease, leases on one bucket name do not overlap, and a lease cannot be taken over while any request of the walk that held it could still be accepted by the backend. Find: a provider call made on a context that does not carry the fence; a client other than the two built with the fenced signer; a request AWS would accept more than fifteen minutes after its stamp (presigned URLs, a retry that re-signs with the current time, a request type that is not SigV4-signed, the SDK's clock-skew correction, a clock on this host running behind); a lease taken over early; a walk that ends cleanly with a call still outstanding; the lease object left in a state nobody can take; a walk on one bucket name that changes another name's state. `TestFenceAgainstAWS` records what was observed on AWS; say whether what it checked is enough to support the claim for the calls the walks make (it checked `ListObjectsV2` and `iam:GetUser`, not a delete). |
| +- **The teardown record** (`TeardownBlock`, `DestroyedAt`, `Leaving`, `ScannerOwed`, the grant that voids `BucketDeletedAt`). Find: a bucket destroyed while another store holds it; a bucket, user or scanner plan left that no record will ever clean up; a store reported healthy on credentials that do not exist; a retained teardown that can never finish, or one that finishes while another account can still read its objects. |
| +- **The simulation** (`internal/objstore/walk_sim_test.go`). Say what it cannot generate: which interleavings, crash points and failure kinds are outside its model, and whether any invariant it states is weaker than the property the code needs. Two faults were found in its bookkeeping and one in the code during this round; look for a third of each. |
| +- **The send lease and the key fence** (`internal/console/billingquantity.go`, `billingservice.go` `reconcileBillingSubscription`, `internal/billingstripe/client.go` `FenceQuantityKey`). The invariant claimed: no request is issued outside a send lease; an operation is acknowledged only when confirmed by the provider or fenced there; a retirement is decided on a provider read taken after the fence; a key is never used for two operations. Find: an acknowledgement that bypasses `quantityOperationSettled`; a fence taken while a sender is live in a way that harms; an operation that can never leave the outstanding state without a person and files nothing; an addition cleared with no hold when nobody can show its operation ran; a quantity lowered with nothing filed. |
| +- **The evidence document.** Check each claim about Stripe against what you know of Stripe's documented behaviour. It says the fence is bounded by key retention, unverified against Stripe, and that a mismatch's HTTP status is not documented. Find any place the code, a comment, a test name, the handoff or `OUTSTANDING.md` says more than that: "proven", "cannot", "never" where the support is a fake. Find any Stripe answer the code would misread as "fenced". |
| +- **The gate.** `CONSOLE_BILLING_QUANTITY_FENCE` and `CONSOLE_BILLING_PRORATION`. Find a way the worker sends anything to the provider with either unset or mis-set, including the fence request itself and the paths that close an ended subscription. |
| +- **`TestSandboxQuantityFence`** (`internal/billingstripe/sandbox_fence_test.go`) has not been run. Read it as the procedure an operator will follow: would a pass actually establish what the gate needs, could it pass against a provider that does not fence, could it touch live mode or print a secret? |
| +- **`OUTSTANDING.md` and the handoff.** Anything stated as reviewable, tested or proven that is not; any launch blocker missing from the list. |
| + |
| +## What to return |
| + |
| +1. The disposition table. |
| +2. New findings, most severe first, in the round 1 format. |
| +3. Claims the code contradicts, in `implementation/carloku-paid-launch/AGENT_HANDOFF.md`, `OUTSTANDING.md` and the evidence document. |
| +4. A one-line verdict: accept, accept with fixes, or reject. |
diff --git a/docs/superpowers/specs/OPUS-REVIEW-carloku-paid-launch-overnight-round6.md b/docs/superpowers/specs/OPUS-REVIEW-carloku-paid-launch-overnight-round6.md| new file mode 100644 |
| index 000000000..71aa2c60b |
| --- /dev/null |
| +++ b/docs/superpowers/specs/OPUS-REVIEW-carloku-paid-launch-overnight-round6.md |
| @@ -0,0 +1,185 @@ |
| +# Opus production and authority review, round 6: Carloku paid launch, `aed42815..058ea506` |
| + |
| +I read `daa32cb6..058ea506` as whole files at `058ea506`. The named tests pass on an exported copy in `$TMPDIR`. I checked the disposition claims by mutating that copy (34 mutations), and wrote three scratch reproductions for the findings below. All scratch work was deleted afterwards. Nothing in the repository was modified and no network command was run. The working tree has uncommitted changes from another session; I did not read them. |
| + |
| +**Production authority** |
| + |
| +- **Attestation stays off without the switch.** The only production path to `SetAttestation` is `applyScope`. It is reached only through `p.attest`, which only `enableScopedAttestation` sets. That is called from one place, `agent.go:258`, behind `scopedAttestationEnabled(os.Getenv)`. The value comes from `EnvironmentFile=-/etc/carlos/host.env`, which is root-owned and 0600, and no code writes it. The ledger opens with `DurabilityLedgerOptions{}` (`durability.go:54`). With no handoff token, no earlier run's attested session is inherited across a restart. With the switch unset, nothing attests. |
| +- **Billing:** billing is off and `CONSOLE_BILLING_PRORATION` is unset. I found no reachable path that bills a customer wrongly. |
| +- **Leaks:** the new error strings name another account's sqid and app (`ErrBucketHeld`, `ErrBucketRetained`). They are only logged; members see fixed copy (`appsettings.go:1038`, `ownership.go:452`, `apistore*.go`). I found no leaked identifier or secret. |
| +- **Customer data:** I walked every crash point and every interleaving the brief names (grant, teardown, re-run and rotate). I found no path that destroys a bucket another app holds. One new path leaves a re-granted store on a deleted key (N2). |
| +- **Not ready for the switch:** the attestation path is not ready to be switched on, even on dev. See N1, N3, N4 and N5. |
| + |
| +## 1. Disposition table |
| + |
| +All paths are at `058ea506`. "Fails without" means I removed or inverted the fix and the named test failed. |
| + |
| +| Finding | Status | Fix | Reproduction; fails without the fix? | |
| +| --- | --- | --- | --- | |
| +| Astra F1 / Opus N4: a grant lands between the teardown's decision and its delete | **Closed** | `objstore/ownership.go:336-339` (grant refused under `Destroying`), `:430-492` (`beginBucketTeardown`), `walk.go:605`, `:645-675` | `TestGrantAndTeardownCannotBothWin` (`ownership_test.go:1124`) and `TestTeardownHoldsTheNameUntilItHasFinished` (`:769`) **fail** without the refusal. Two parts are **not pinned**: removing both `confirm()` re-reads passes, and removing "defer to another app's `Destroying` marker" (`ownership.go:463`) passes. Both are second lines of defence. | |
| +| Astra F1 (legacy part) / Opus N2: a holder with no interval | **Closed** | `walk.go:782-828` (`otherStoreHolders`), `ownership.go:452-459` | `TestTeardownFindsALegacyHolderByItsRecord` (`:1048`) **fails** without it. | |
| +| Astra F3 A: two concurrent sharers each defer to the other | **Closed** | `ownership.go:450`, `:471` (`Leaving`) | `TestConcurrentTeardownsOfASharedBucketLeaveExactlyOneDestroyer` (`:1082`) **fails** with `Leaving` not set, and with `Leaving` not honoured. | |
| +| Astra F3 B / Opus N3: the scanner plan leaks | **Closed** | `ownership.go:473-475` (`ScannerOwed`), `walk.go:632` | `:905`, `:993` and `:1082` **fail** without it. | |
| +| Opus N1: a cross-account skip keeps the data and reports success | **Closed** as asked. The new `ErrBucketRetained` state has its own defects; see N2. | `ownership.go:354-356` (`ErrBucketHeld`), `walk.go:718-720`, `:595`, `:678-687` (`TeardownBlock`) | `TestMintingGrantIsRefusedWhileAnotherAppHoldsTheName` (`:870`) **fails** without `ErrBucketHeld`. `:905` **fails** without `Retained`. `:769` **fails** without `bucketDone`. | |
| +| Astra F4: completeness dated at the caller's clock | **Closed** | `walk.go:148`, `:233` | `TestCompletenessIsDatedAfterTheProbesAnswer` (`:1167`) **fails** with `observed = now`. All three console callers pass `time.Now()` after taking the lock. | |
| +| Astra F2: retiring a future-dated operation under its sender | **Closed**. A residual window is narrowed, not closed; see "Other answers". | `console/billingquantity.go:795-802` (not retired), `:698-704` (sender re-read) | `TestQuantityFutureDatedOperationIsNotRetiredUnderItsSender` (`billingquantity_test.go:1205`) and `TestQuantitySenderDoesNotSendARetiredOperation` (`:1265`) each **fail** without their fix. | |
| +| Astra F9 / Opus N8: a coincidental quantity is backdated and labelled `charged` | **Closed** | `billingquantity.go:727` (confirm), `:785-791` (label), `billingservice.go:514-516` (wait), `billingstore.go:453-472` | `:1032`, `:1064`, `:996` and `:295` **fail** when the store, label or wait gating is removed. Removing the `confirmQuantityOperation` call fails the worker tests. | |
| +| Opus N5: the clamp dates a proration before the addition | **Closed** | `billingquantity.go:639` (clamp), `:405-413` (compare-and-set) | Without the clamp, `TestQuantityOperationIsNotDatedBeforeTheAdditionItChargesFor` (`:1286`) **fails**. Without the compare-and-set clause, `TestBeginQuantityOperationIsACompareAndSet` **fails**. | |
| +| Astra F10 / Opus N9: the guard is dropped and recreated outside a transaction | **Closed** | `billingquantityaudit.go:33-41` (`BEGIN IMMEDIATE` … `COMMIT` in one `Exec`) | `TestQuantityProcessingGuardIsReplacedWhenItsBodyChanges` (`:1473`) passes. Replacing the guard unconditionally also passes, which is harmless. The rollback consequence is in the handoff. | |
| +| Astra F8: the absent-month memory never expires | **Closed** | `economicspass.go:1404` (6-hour TTL) | `TestStatementCatchUpRevalidatesAnAbsentMonth` (`meterfolds_test.go:926`) **fails** without the TTL. | |
| +| Opus N6: one unreadable statement hides its month | **Closed** | `economicspass.go:1423-1440` | `TestStatementMonthStateEdges` (`:850`) **fails** with the rule removed, and **fails** with the rule applied to the platform statement. Unknown is now pinned. | |
| +| Astra F5: the registration tolerance erases coverage | **Closed** | `durabilityfold.go:75-112`, `:141` | `TestDurabilityFoldProvesTheUnionOfABoxsWindows` (`meterfolds_test.go:562`) **fails** without the metrics and edge widening. The fold asks for the hull of the windows, not their union. That is stricter, so a box with two disjoint windows in one day holds. | |
| +| Astra F6 / Opus N7 (first match, nesting, duplicates, links) | **Closed as asked; links are only checked at the moment of the tick (N4)** | `durabilityscope.go:150-158` (no `break`), `:131-135`, `durabilityreplicator.go:125-137` | `TestDurabilityScope` **fails** with `break` restored, and with route resolution removed. `TestReplicatorProbe` **fails** without config-path resolution. | |
| +| Astra F7 / Opus N7: unit running with no config | **Closed** | `durabilityscope.go:107-109`, `durabilityreplicator.go:87-89`, `:97-99`, `:167-169` | `TestDurabilityScope` **fails** without the scope rule. The reader's own copy of the rule is unpinned, but the scope covers it. `TestReplicatorProbe` **fails** without the command check. | |
| +| Opus N7: torn reads and how far back an exclusion reaches | **Torn reads closed. Reach-back over-corrected (N1).** | `durabilityreplicator.go:138-161`, `:175-187`; `durabilityscope.go:259-267` | Removing each re-check (content, mtime, unit) makes `TestReplicatorProbe` or `TestReadDurabilityScopeIsOneObservation` **fail**. `TestScopedAttestationExcludesFromWhenTheConfigWasWritten` **passes** with the config-time term removed, because its previous tick (23:00) already precedes the write (23:58). It fails only when the config term and the `verified` term are both removed. Four parts of `applyScope` are unpinned: attest before seal (moving `attest` after `Seal` passes), ending at `verified`, the escape fallback, and the config term on its own. | |
| +| Astra F11: the plan misstates its boundaries | **Closed (documents)** | plan lines 17 and 31-34; `billingcalculation.go:143-149` | none needed | |
| +| Opus round 5: the dating's tolerance and frozen guards are unpinned | **Closed** | tests | `TestQuantityConfirmedLandingIsNotDatedAheadOfTheClockOrIntoAFrozenMonth` (`:1124`) **fails** with either guard removed. | |
| + |
| +The switch test `TestScopedAttestationIsOffUnlessAskedForByName` **fails** if any non-empty value enables attestation. It **passes** with `strings.Contains(v, "scoped")`. It does not cover the `agent.go` wiring. The code itself is exact. |
| + |
| +## 2. New findings, most severe first |
| + |
| +### N1. P1 (behind the switch): a new exclusion is dated back to the config's last write, which rewrites days already published complete. The refusal fallback makes such a day incomplete for every account on the box. |
| + |
| +**Where:** `internal/host/durabilityscope.go:259-267`, with `usage/durabilityledger.go:894-905` (`insertSession` touches every day it reaches) and `:1563-1566` (`Escaped` latches a failure on the instant it is given, before any clamp). |
| + |
| +**The cause:** `from = min(verified, ConfigModified)`. When `ConfigModified < verified`, the previous tick already read this same config, and that config did not name the account; otherwise the account would already be excluded. The config term therefore adds nothing true. A new exclusion that comes from the registry alone (a route stops hibernating, or a new always-on exec route; the config is only regenerated when an operator runs `carlos ops litestream`) is still dated at the config's last write. That can be days or weeks ago. |
| + |
| +**Scenario A (reproduced):** |
| + |
| +1. The first claim is on the 16th at 23:00. The config is rewritten on the 17th at 06:00, and the 17th is sealed and published complete for `bac`. |
| +2. On the 18th, `bac`'s route stops hibernating. The config does not change. |
| +3. The exclusion is dated the 17th at 06:00. The new session touches the 17th, which republishes as incomplete for `bac`. |
| + |
| +**Scenario B (reproduced):** |
| + |
| +1. Same setup. A single `systemctl` failure on the 18th ends the claim, and the claim restarts at the 18th, 12:00. |
| +2. On the 19th, `bac` flips. `from` (the 17th) now falls before the session start, so `SetAttestation` refuses it. |
| +3. `Escaped(bac, 17th 06:00)` latches a recording failure on the 17th. The 17th republishes as incomplete for **every** account on the box, including `bae`, which never changed. |
| + |
| +**Consequence:** a day already folded into a closed statement changes. That statement's pinned evidence no longer verifies (`Statement.BillingSnapshot` returns `ErrEvidenceChanged`), and there is no rebuild door, so the month holds. Because the month's evidence is one shared snapshot, it can hold for accounts beyond this box. The comment's claim "blunter, and never more generous" is also false the other way: `Escaped` clamps the session to its start, so on a later day the gap `[from, sessionFrom)` is not excluded. |
| + |
| +**Smallest fix:** |
| + |
| +- Date a new exclusion at `verified` and drop the `ConfigModified` term. Any config written after the previous verified tick is either seen as stale (`UnitStarted < ConfigModified`, which attests nothing) or post-dates `verified`. The 23:58 case still closes correctly, because attest runs before seal. |
| +- In the refusal path, call `Escaped` at `max(from, sessionFrom)`, never earlier than the last verified tick. |
| +- Pin the fix with a test that flips a route through the registry alone, with an old config, after a day has closed. |
| + |
| +### N2. P2: after `ErrBucketRetained`, a re-grant reports the store healthy on a deleted key, the member is told something false, and nothing ever finishes the teardown |
| + |
| +**Where:** `walk.go:718-720`, which returns before the delivered variables, the scan sidecar or the record are touched. Then `walk.go:162`, where `rebuilt` is driven only by `rec.Teardown`, and `walk.go:276-283`, the short-circuit. |
| + |
| +**Scenario (reproduced):** |
| + |
| +1. `bac/one` and `bad/two` both adopt one bucket. |
| +2. `Teardown(bac/one)` returns `ErrBucketRetained`. The user and keys are deleted, but `Provisioned` and the delivered variables remain. |
| +3. A re-grant of `bac/one` finds the variables present. `EnsureUser` recreates the user with no key, and `Grant` returns the record as healthy, naming `MEMKEY1`, a key that no longer exists. |
| +4. `regrantForScanning` (`apiscan.go:541`) does the same whenever scanning is toggled. It also reopens the interval. |
| + |
| +**What the member sees:** the store door says "it is still there; try again once the backend is reachable" (`appsettings.go:1043`). The purge door says "nothing was deleted; try again" (`ownership.go:452`). Both are false, since the credentials are gone, and retrying never helps. The app cannot be purged while the record stands. |
| + |
| +**Never finishes:** suppose the other store's teardown destroys the bucket, and then the name is granted again (the other app's minted name, or a new adopter) before `bac/one` re-runs. The re-run then finds the new holder and returns `ErrBucketRetained` again, indefinitely. Yet `bac/one`'s objects were destroyed with the first bucket. |
| + |
| +**Operator way out:** there is none in code. The only exits are tearing down the other store, or deleting `store.json` (plus the variables and sidecar) by hand. |
| + |
| +**Smallest fix:** |
| + |
| +- On `Retained`, record `TeardownBlock{UserDeletedAt}` so that `Grant` takes the `rebuilt` path. |
| +- Have the destroyer stamp the ledger with `DestroyedAt`. A teardown whose own interval closed before the latest `DestroyedAt` then finishes quietly. |
| +- Give `ErrBucketRetained` its own member copy. That copy needs Paul's review. |
| + |
| +### N3. P2 (behind the switch): a restart or config change between ticks is not noticed, so an account the replicator streamed in between is vouched for |
| + |
| +**Where:** `durabilityscope.go:225-268`. Each tick compares only its own snapshot. |
| + |
| +**Scenario:** |
| + |
| +1. Between two ticks, `bac` goes always-on, the config is regenerated to name it, and the unit restarts and streams `bac`. |
| +2. Then `bac` goes back to hibernating, the config is regenerated without it, and the unit restarts again. |
| +3. The next tick sees a config without `bac`, a unit that started after the config was written, and a hibernating route. `bac` is vouched for over an interval in which the external replicator streamed its database. The same happens if the unit is stopped instead of restarted the second time. |
| + |
| +**Smallest fix:** if `UnitStarted` or `ConfigModified` is later than `verified`, end the claim at `verified` and start it again at `now` with the current scope. Ambiguity then holds the day. It costs one held day per regeneration, which a regeneration already costs whenever a tick lands mid-restart. |
| + |
| +### N4. P2 (behind the switch): symlinks are resolved when the tick reads them, not when the replicator opened the file, and a path that does not exist is accepted |
| + |
| +**Where:** `durabilityreplicator.go:126-136`. |
| + |
| +**Scenario:** |
| + |
| +1. A unit tenant `B` links `instances/y.db` to `/data/y.example.com.db`, the hibernating database of account `A`. |
| +2. The generator names the link, and litestream opens it and holds `A`'s database open. |
| +3. `B` then removes the link, or points it at its own file. |
| +4. The reader gets `ErrNotExist`, which it treats as "nothing is streamed from it now", or a harmless target. The path is attributed to `B` alone, and `A` is vouched for while the replicator still has `A`'s database open. |
| + |
| +This contradicts the plan's "refuses … a path it cannot resolve". |
| + |
| +**Smallest fix:** |
| + |
| +- Treat `ErrNotExist` as an error. |
| +- Refuse attestation when a configured path's resolved form differs from its written form, except for systemd's own `StateDirectory` → `/var/lib/private` rewrite. |
| +- A stronger check: read the unit's `MainPID` from `/proc/<pid>/fd` and match the files it actually holds. |
| + |
| +### N5. P2 (behind the switch): only the systemd unit is inspected, so a second replicator is invisible |
| + |
| +**Where:** `durabilityreplicator.go:82-90` and `:193-255`. |
| + |
| +**Scenarios:** |
| + |
| +- A `litestream replicate <db> <url>` started by hand, in tmux or from cron. |
| +- A unit ported from titogo or Eleven. |
| +- An `ExecStartPost=` drop-in that starts one in the background. Only `ExecStart` is counted. |
| +- `LoadState=not-found` reads as "plainly not running", so a box with no unit but a replicator running outside systemd attests. |
| + |
| +**Smallest fix:** refuse attestation when any process on the box whose `exe` or `comm` is litestream is not the unit's `MainPID`, or lies outside its cgroup. The agent can read `/proc`. |
| + |
| +### N6. P3 (holds): every agent restart holds that day for every account on the box, and so does a box's first day |
| + |
| +**Where:** `durability.go:54`, which presents no `HandoffToken`. `durabilityledger.go:796-817` therefore files a gap on every open. With the switch on, the window between the open and the first tick is also unattested. |
| + |
| +**How often:** every roll, `carlos system update`, crash or reboot, plus the box's first day and the day the ledger build first rolls out. Completeness needs every day of the month, so a box that restarts its agent once a month never completes `durability_writes_1k` for any account. This is a correct hold, not a wrong bill. The week on dev will show it, and nothing so far says it. |
| + |
| +**Fix:** none is required for correctness. Persist the clean-stop token across restarts, for example in `/run/carlos`, if restart days should ever count. |
| + |
| +### N7. P3 (holds): the agent cannot resolve a unit tenant's paths, so any box whose config names one never attests |
| + |
| +**Where:** `durabilityreplicator.go:127`. The agent runs as `User=carlos` (`carlos-edge.service:28`). `StateDirectory=` under `DynamicUser=yes` lives in `/var/lib/private`, which is mode 0700 and owned by root. `EvalSymlinks` returns `EACCES`, which is treated as an error, so nothing is attested. |
| + |
| +The flagship's `kass` is such a tenant. This holds, and the flagship already holds for the hand-kept config. |
| + |
| +Two doc mismatches: `durabilityscope.go:27` and `litestream.go:280` say the unit runs "as root", but `carlos-litestream.service` says `User=carlos`. |
| + |
| +### N8. P3: the config's modification time can lie |
| + |
| +**Where:** `durabilityscope.go:110`. `cp -p`, `rsync -t`, `install -p` or restoring `/etc` from an image can backdate `mtime`. A unit then running an older config passes `UnitStarted ≥ ConfigModified`. |
| + |
| +**Fix:** use `max(mtime, ctime)`. Userland cannot set `ctime`. |
| + |
| +### N9. P3: `GeneratedLitestreamDBs` accepts `$`, which litestream expands |
| + |
| +**Where:** `ops/litestream.go:205`, the pattern `(/\S+)`. `litestream replicate` expands environment variables in its config by default (`third_party/litestream/cmd/litestream/replicate.go:87`), and `expectedReplicatorCommand` forbids `-no-expand-env`. The reader matches the literal path while litestream opens the expanded one. |
| + |
| +The generator copies tenant-chosen file names from unit trees (`treeDBs`). A file named `${LITESTREAM_SECRET_ACCESS_KEY}.db` therefore puts that variable into litestream's path and replica key. That generator behaviour is outside this diff but worth its own thread. |
| + |
| +**Fix:** refuse `$` in every line. The generator should skip such names. |
| + |
| +## 3. Handoff and plan claims the code contradicts |
| + |
| +- **Attestation plan, "A new exclusion is dated no later than the config's writing … so a day cannot close complete…":** it is dated earlier than either needs. On the registry-only path it rewrites closed, published days, and through the fallback it makes them incomplete for everyone (N1). |
| +- **Attestation plan, the reader "refuses … a path it cannot resolve":** a path that does not exist is accepted (N4). |
| +- **Dispositions, `TestScopedAttestationExcludesFromWhenTheConfigWasWritten` pins dating at the config's writing:** it passes with the config term removed. |
| +- **Dispositions, "pins that only the exact value turns it on":** the test passes with a `strings.Contains` switch, and the `agent.go` wiring is untested. The code is exact. |
| +- **Dispositions and handoff item 5, a retained teardown "keeps its record … until the other store is torn down or a person removes the bucket":** a re-run can stay `Retained` after that store is gone, and a re-grant reports a dead key healthy (N2). |
| +- **Handoff item 3 and the "Before it may be set anywhere" list:** they do not say that every agent-restart day, and a box's first day, hold for everyone (N6), or that boxes with unit tenants cannot attest (N7). Both shape what the dev week will show. |
| +- **`durabilityscope.go:27`, "run as root":** the unit runs as `carlos`. |
| + |
| +### Other answers |
| + |
| +- **The quantity sender's re-read** narrows the stale-sender window but does not close it. Another process can retire the operation, for example on a mismatch after a hand change, between the re-read (`billingquantity.go:700`) and the provider executing. That window is one round trip. |
| +- **The webhook wait** is bounded by `CreatedAt` plus 2 minutes. A webhook for an unrelated change is not swallowed: the 503 is retried and the reconcile re-reads the whole subscription. A sender whose provider answer lands after 2 minutes, which is possible within the client's timeout, gets an observation date and a review. That holds; it does not misdate. |
| +- **`NewestAt`** can only push the date later. A date at or past the period end records nothing, and the additions then go stale and get a review. No date is refused on every pass beyond clock skew. |
| +- **`statementMonthState`:** the unread counter counts reads, not passes. A transient failure of three reads classifies an open statement as present. That account's month is then not swept until the read recovers, at which point it heals. No bill results. |
| +- **A week on dev cannot show:** N1 (it needs a statement closed over the rewritten day), N3, N4, N5, N8 and N9. N6 it will show at once. |
| + |
| +## 4. Verdict |
| + |
| +**Accept with fixes.** Nothing in production attests without `CARLOS_DURABILITY_ATTEST=scoped`, nothing bills, leaks or destroys a held bucket, and both flags stay unset. Before the switch is set anywhere, dev included, fix N1, N3, N4 and N5 and pin `applyScope`'s dating and ordering. Before the teardown guard lands, fix N2. |
diff --git a/implementation/carloku-paid-launch/AGENT_HANDOFF.md b/implementation/carloku-paid-launch/AGENT_HANDOFF.md| index ddc8c57e2..d31ecaa81 100644 |
| --- a/implementation/carloku-paid-launch/AGENT_HANDOFF.md |
| +++ b/implementation/carloku-paid-launch/AGENT_HANDOFF.md |
| @@ -1,6 +1,6 @@ |
| # Agent pickup instructions: Carloku paid launch |
| -CURRENT GATE (2026-10-01): work in progress, pushed, not merged, not deployed. Billing is OFF. Live Stripe is read-only and was not touched in this run. No AWS or production change was made. Nothing here is launch-ready until a real purchase, webhook and metering are verified live. |
| +CURRENT GATE (2026-10-01): work in progress, not merged, not deployed. The pricing page, the console billing page and the signup demo are reviewable. The backend is **not production-ready**: `OUTSTANDING.md` beside this file lists the launch blockers and says, item by item, what is reviewable and what is proven. Billing is OFF. Live Stripe is read-only and was not touched in this run. No AWS or production change was made. Nothing here is launch-ready until a real purchase, webhook and metering are verified live. |
| Read `AGENTS.md`, the spec `docs/superpowers/specs/2026-09-07-carloku-pricing-plans-design.md`, `requirements-audit.json` (the 18 spec steps, none complete) and this file. `session.md` is the September investigation log; prefer this file where they disagree. `.claude/HANDOFF.md` and the Amadan branch handoff carry the short status. |
| @@ -30,12 +30,12 @@ When a review is approved, apply each string byte for byte: the site by replacin |
| 1. Plans and calculations, Stripe boundary, entitlements and membership, quantity history, invoice engine: as described in the September handoff (kept in git history at `aed42815`). Unchanged in this run. |
| 2. Console GET `/a/{sqid}/billing` (`billingpage.go`, `templates/billing.html`): current plan, owner-only chooser posting to the existing checkout door, `?country=` price preview computed from the plans terms, error and pending banners, portal button once a customer exists, read-only for members, byte-identical 404 for non-members, closed state when billing is off. `plans.Countries()` exposes the embedded country list. |
| -3. Durability ledger: the epoch read race is fixed (epoch and created-at are immutable after recover) with a `-race` regression. An unmetered replication fallback on a box whose ledger opened now files an escape (`DurabilityLedger.Escaped`): the box-day is incomplete and the owner is excluded for the run. A box whose ledger failed to open has no publisher and files nothing. The console fold asks each box's published day to prove the union of that box's durability, metrics and edge windows; a box with no ledger directory, no day file, or a ledger whose coverage starts after the box began serving holds the day for everyone. There is no tolerance. Durability days are counted per account: a box-day attested with an exclusion set is complete for every account the box did not exclude. The move worker's checkpoint is recorded against the route's owner. Scoped attestation is built end to end and is OFF: the agent attests only with `CARLOS_DURABILITY_ATTEST=scoped` in its environment, which is set nowhere. Attestation is still false everywhere in production. |
| +3. Durability ledger: the epoch read race is fixed (epoch and created-at are immutable after recover) with a `-race` regression. An unmetered replication fallback on a box whose ledger opened now files an escape (`DurabilityLedger.Escaped`): the box-day is incomplete and the owner is excluded for the run. A box whose ledger failed to open has no publisher and files nothing. The console fold asks each box's published day to prove the union of that box's durability, metrics and edge windows; a box with no ledger directory, no day file, or a ledger whose coverage starts after the box began serving holds the day for everyone. There is no tolerance. Durability days are counted per account: a box-day attested with an exclusion set is complete for every account the box did not exclude. The move worker's checkpoint is recorded against the route's owner. Nothing attests: attestation is false everywhere, in this code and in production. The scoped attestation (path A) was built, reviewed in round six, found unable to prove an exclusive writer, and removed from this branch. It is kept on `feat/carloku-scoped-attestation-20261001` with its findings open. Replication-write completeness is a launch blocker, not a waived requirement. |
| 4. `internal/awsbilling`: Cost and Usage Report reader (versioned deliveries only), newest-delivery selection, digests, charged S3 transfer classifier. It holds on a charged usage line it does not count and on any non-usage line (fee, refund, credit, discount) with a cost on a transfer usage type. Every transfer usage type the flagship account wrote from July to September 2026 is pinned in a test. |
| -5. `internal/objstore/ownership.go`: append-only bucket ownership ledger, opened at grant, closed at teardown. It resolves an owner only for a ledger that claims completeness, and only for hours that begin at or after the instant it began vouching, so a store's partial first hour never resolves (a ledger that began vouching exactly on the hour answers for that hour). Completeness needs a grant block recorded as minting, nothing delivered to the app yet, a probe showing the bucket does not exist, and an inventory showing no principal's policy names the bucket (`GranteeProber`), and is dated when both probes had answered. The AWS provisioner cannot list principals yet, so on AWS no ledger claims completeness, new stores included, and store transfer holds. A minting grant is refused while another app holds the name. Teardown decides by compare-and-swap on the ledger: if nobody else holds the bucket (by ledger interval or by store record) it marks the bucket as being destroyed, which refuses every grant of the name until it has finished; otherwise it leaves the bucket and its scanner, revokes its own credentials, and records that the scanner is owed. A teardown that leaves a bucket to another account keeps its record and returns `ErrBucketRetained` (discussion 191). |
| +5. `internal/objstore/ownership.go`: append-only bucket ownership ledger, opened at grant, closed at teardown. It resolves an owner only for a ledger that claims completeness, and only for hours that begin at or after the instant it began vouching, so a store's partial first hour never resolves (a ledger that began vouching exactly on the hour answers for that hour). Completeness needs a grant block recorded as minting, nothing delivered to the app yet, a probe showing the bucket does not exist, and an inventory showing no principal's policy names the bucket (`GranteeProber`), and is dated when both probes had answered. The AWS provisioner cannot list principals yet, so on AWS no ledger claims completeness, new stores included, and store transfer holds. A minting grant is refused while another app holds the name by ledger interval or by store record, and for a bucket that already exists and it never held. Grant and teardown are walks, and one walk at a time runs on a bucket name across every process: a lease object taken by compare-and-swap (`walklock.go`), bound to the invocation, standing until it runs out if the walk failed. Every request of a walk is signed with the instant its lease was taken (`fence.go`), so AWS refuses a request a paused process sends after its lease could be taken over. That was checked read-only against S3 and IAM (`TestFenceAgainstAWS`). A teardown writes a block on its own record before it touches anything and decides by compare-and-swap on the ledger: if nobody else holds the bucket it marks the bucket as being destroyed; otherwise it leaves the bucket and its scanner, revokes its own credentials, and records that the scanner is owed. A grant by the same app over its own unfinished teardown takes the store back, runs in full, and voids the teardown's note that the bucket was deleted. A teardown that leaves a bucket to another account keeps its record and returns `ErrBucketRetained` (discussion 191), and finishes once that bucket has been destroyed. `ErrBucketRetained` has no member copy yet; it needs Paul's review. The invariants are checked by a randomized simulation (`walk_sim_test.go`; 30,000 seeds on 2026-10-01). |
| 6. Statement folds: `console/durabilityfold.go` (`durability_writes_1k`, policy in `usage.BillableReplicationWrites`) and `console/storeegressfold.go` (`store_egress_gb`). `usage.MeterSource` names a source for both meters. Month evidence pins ownership ledgers by their month projection and knows the durability box directory. Statements carry the five billed meters by UTC day (`Statement.Days`), summing to the monthly totals; the other meters are not recorded by day. With a transfer report configured, the statements catch-up walks a 36-month window and sweeps every month that has an open statement, is missing a statement an account is owed (the platform's included), or is a gap between metered months. A statement that will not parse, or that cannot be read three passes running, is reported once per process and the month is classified without it. A month seen with no statement is remembered for six hours, a closed statement for the life of the process. |
| 7. Operator API doors (`billingoperator.go`): list and resolve billing review holds, list and set the house marker. No CLI verb yet. |
| -8. Team quantity worker (`billingquantity.go`) behind `CONSOLE_BILLING_PRORATION` (`immediate` or `next_invoice`). Unset, which is how it must stay until Paul decides, it makes no provider call. An operation is recorded by a compare-and-set on the billing row's revision, subscription, period, quantity and status, on the target and newest pending addition recomputed from the database, and on its own instant lying inside the period and not before the addition it charges for. The request dates its proration at that instant (`proration_date`). A sender re-reads its operation before sending and does not send one that has been retired; an operation dated ahead of the observing clock is left alone. An operation is confirmed only by the provider's success answer to its own request. A confirmed landing takes effect at the operation's instant, so a seat's charge and its allowance start together; a quantity that merely matches an unconfirmed operation's target is recorded as `landed_unverified`, dated when observed, with a review. If the provider refuses a request and names `proration_date`, the operation is retired at once; if it does not name it, the operation waits out the 22-hour replay limit. None of the provider's side is confirmed: test mode is not authorized. Only additions of the subscription's current period can raise it; older ones are closed uncharged with a review; additions and operations of an ended subscription are closed. It does not yet lower the quantity at renewal. |
| +8. Team quantity worker (`billingquantity.go`) behind two switches, both unset: `CONSOLE_BILLING_PRORATION` (`immediate` or `next_invoice`, Paul's decision) and `CONSOLE_BILLING_QUANTITY_FENCE=stripe-sandbox-verified` (set only after `TestSandboxQuantityFence` has passed against a Stripe test-mode account; it has not been run). With either unset it makes no provider call. An operation is recorded by a compare-and-set on the billing row's revision, subscription, period, quantity and status, on the target and newest pending addition recomputed from the database, and on its own instant lying inside the period and not before the addition it charges for. The request dates its proration at that instant (`proration_date`). Sending and retiring exclude each other through a send lease on the operation's row; an operation dated ahead of the observing clock is left alone. An operation nobody can show ran is acknowledged only after its idempotency key has been consumed at the provider with a metadata-only request (the fence), and a retirement is decided on a provider read taken after that. The fence rests on Stripe's documented key rules, is modelled by a fake in the tests, is not verified against Stripe, and lasts only as long as Stripe remembers a key (documented as at least 24 hours): `docs/superpowers/specs/2026-10-01-stripe-idempotency-evidence.md`. A quantity that goes down inside a period files `billing_quantity_lowered`. An operation is confirmed only by the provider's success answer to its own request. A confirmed landing takes effect at the operation's instant, so a seat's charge and its allowance start together; a quantity that merely matches an unconfirmed operation's target is recorded as `landed_unverified`, dated when observed, with a review. If the provider refuses a request and names `proration_date`, the operation's key is fenced at once and it is retired on the next read; if it does not name it, the operation waits out the 22-hour replay limit. None of the provider's side is confirmed: test mode is not authorized. Only additions of the subscription's current period can raise it; older ones are closed uncharged with a review; additions and operations of an ended subscription are closed. It does not yet lower the quantity at renewal. |
| 9. Stripe provisioning plan `stripe-provisioning.md` and print-only `stripe-provision.sh` (verified never to invoke the CLI). |
| 10. Partial-month billing: `plans.AllowanceForMonth` returns the paid spans of a month and the allowance weighted over them. A partly covered month (a first month, a month a subscription ends in, a lapse) bills only the UTC days wholly inside a paid span, against that allowance, and records the days it counted. Completeness is still whole-month. **The whole-paid-day rule is not in the approved spec, which speaks only of prorating the allowance. It is a proposal that needs Paul's approval; the plan document states what it rounds away and the tests that pin its boundary.** A month with a paid span and no whole paid day bills nothing. A month with no paid span is refused quietly. Two subscriptions in one month price and then hold on `billing_invoice_identity_mixed`. |
| 11. Plans: `docs/superpowers/plans/2026-10-01-store-transfer-cur-collector.md`, `2026-10-01-durability-attestation-path.md`, `2026-10-01-bucket-ownership-attestation.md`, `2026-10-01-partial-month-billing.md` (built). |
| @@ -45,7 +45,7 @@ When a review is approved, apply each string byte for byte: the site by replacin |
| The monthly freeze requires all five billed meters complete. Two cannot be, in production, today: |
| - `store_egress_gb`: no Cost and Usage Report export exists, so the source is unconfigured. It needs the scoped infrastructure approval in the collector plan, a first real delivery, a principal inventory in the AWS provisioner, and attested ownership for the stores that already exist. A read-only look at the account's billing on 2026-10-01 (collector plan) found no export, no credit or discount line on S3 from July to September, no free transfer allowance reaching the account, and 1,395 GB of inter-region transfer out of Ireland in September that looks like the platform's own bucket. |
| -- `durability_writes_1k`: no box attests, because the external `carlos-litestream` binary replicates unit-backed and always-on routes outside the ledger. The chosen way out is the scoped attestation (attestation plan, "Decision and progress"), built end to end behind a switch that is off. Before it may be set anywhere: an independent review of the call and the reader, a week on a dev box, the flagship's hand-kept litestream config cut over to the generated one, and a scoped approval. Do not set the flag to pass a test. |
| +- `durability_writes_1k`: no box attests, because the external `carlos-litestream` binary replicates unit-backed and always-on routes outside the ledger. The scoped attestation (path A) was the first way out. Round six showed it cannot prove an exclusive writer (a config's modification time is not the running configuration, and one named unit is not the inventory of writers), so it is off this branch. What can prove the meter is path B in the attestation plan: the platform's own replicator as the only writer, counting what it writes. It is not built. Until it is, every month's replication meter is incomplete and nothing is invoiced. Do not enable any attestation to pass a test. |
| ## Still missing from the full requirements audit |
| @@ -60,6 +60,8 @@ The monthly freeze requires all five billed meters complete. Two cannot be, in p |
| - An operator verb to backfill bucket ownership for existing stores: missing. |
| - A store's partial first hour holds the month if it has a charged line, and nothing releases the hold. Needs a decision before the AWS principal inventory lands (collector plan). |
| - A box whose ledger opened after the box began serving holds that day for everyone. Whether a box's first minutes may be waved through is undecided, so they hold. |
| +- The quantity fence is unverified against Stripe and is not a permanent token (item 8). A sender frozen for more than a day can still land a request; that is detected and held, not prevented. |
| +- The invoice engine's lease has no provider-side fence beyond identical replays under one key. Past Stripe's key memory a late request runs as new and is detected and held. One schedule leaves an open invoice with a wrong total that the engine refuses to charge; whether the customer could pay it from Stripe's hosted page is not determined. |
| - Rolling back to a build from before `5f2b4faf` on a database that has recorded a `landed_unverified` outcome marks those accounts `history_unproven`, a durable hold: the older build's audit sweep does not know the outcome. Nothing has recorded one (the proration switch is unset). |
| ## Verification in this run |
| @@ -75,7 +77,8 @@ Merchant `acct_1UD7LXI4Nrr6AEsO`, Team Tito Limited, Ireland, EUR; Irish standar |
| ## Next actions, in order |
| 1. Apply approved copy from both reviews when `result.json` arrives. |
| -2. Independent review: Astra rejected rounds 1 to 5 (ten, eight, five, five and eleven findings); Opus accepted with fixes four times (eight, eight, eight, nine). All were accepted and fixed with reproductions (`docs/superpowers/specs/CODEX-DISPOSITIONS-carloku-paid-launch-overnight.md`). Round six of both must pass on the exact tip before anything lands; do not read "fixed" as "accepted". The agent's attestation call and the rewritten teardown decision have had no independent review yet. |
| +2. Independent review: Astra rejected rounds 1 to 6 (ten, eight, five, five, eleven and fourteen findings); Opus accepted with fixes five times (eight, eight, eight, nine, nine). The storage and quantity findings were fixed at the mechanism with simulations and mutation checks; the attestation findings are open against the parked branch (`docs/superpowers/specs/CODEX-DISPOSITIONS-carloku-paid-launch-overnight.md`, "Round six"). Round seven of both must pass on the exact tip before anything lands; do not read "fixed" as "accepted". The walk lease, the backend fence, the send lease and the key fence have had no independent review yet. |
| 3. Scoped approvals: Cost and Usage Report export; Stripe catalog, portal, webhook and the `rk_live` key; then a test-mode rehearsal and a real purchase. |
| 4. Build the principal inventory and the bucket ownership attestation (designed in `docs/superpowers/plans/`), quantity reduction at renewal, a `carlos` verb and documentation for the operator doors, then Free-tier enforcement. |
| -5. Take the scoped attestation through its review, a week on dev and the flagship's config cutover. Until then replication writes hold every month. |
| +5. Build path B of the attestation plan (the platform's own replicator as sole writer). Until a box can prove its replication writes, that meter holds every month. |
| +6. Run `TestSandboxQuantityFence` once Stripe test mode is authorized. The quantity worker stays off until it has passed. |
diff --git a/implementation/carloku-paid-launch/OUTSTANDING.md b/implementation/carloku-paid-launch/OUTSTANDING.md| new file mode 100644 |
| index 000000000..1b4142146 |
| --- /dev/null |
| +++ b/implementation/carloku-paid-launch/OUTSTANDING.md |
| @@ -0,0 +1,70 @@ |
| +# Carloku paid launch: what is reviewable, what is proven, what blocks launch |
| + |
| +Written 2026-10-01. Billing is OFF. Nothing here was deployed. No live Stripe write, no AWS change, no charge. |
| + |
| +Three words are used exactly: |
| + |
| +- **Reviewable**: a person can open it and judge it. It says nothing about production. |
| +- **Tested**: automated tests pass on this branch against fakes or fixtures. |
| +- **Proven**: checked against the real system it depends on. |
| + |
| +A passing test against a fake, a switch that is off, and an attestation that excludes the accounts it cannot vouch for are none of them "proven", and none is called complete below. |
| + |
| +## Reviewable now |
| + |
| +| Surface | Where | What it is | What it is not | |
| +| --- | --- | --- | --- | |
| +| Pricing page copy | `http://100.119.180.57:38009` (name form `docs-t1.tail5fd3ba.ts.net:38009`) | 87 strings in a hand-built preview of the page | Not approved copy. The copy approved on 2026-09-22 was lost with `/tmp` and is not claimed | |
| +| Console billing page copy | `http://100.119.180.57:45033` | 40 strings; ten page states rendered through the real handlers | Not approved copy | |
| +| Signup and billing demo | `http://100.119.180.57:41881` | The real console handlers and store. Sign-in, checkout, payment, webhook, active plan, portal | Keymail and Stripe are labelled simulations on the same host. It is not Stripe proof | |
| + |
| +## Launch blockers |
| + |
| +Each is concrete. None is waived. |
| + |
| +### Meters (the complete metered model cannot invoice until both are proven) |
| + |
| +1. **Replication writes (`durability_writes_1k`) cannot be proven on any box.** The external replicator writes outside the ledger. Path A (scoped attestation) was built, reviewed in round six, and cannot prove an exclusive writer; it is off this branch. Path B (the platform's own replicator as the only writer) is designed and not built, and needs a root unit change on every box under a scoped approval. Until then this meter is incomplete every month and nothing is invoiced. |
| +2. **Store transfer (`store_egress_gb`) has no source.** No Cost and Usage Report export exists on the account. It needs the scoped infrastructure approval, a first real delivery, an IAM principal inventory in the AWS provisioner (not built), and the bucket ownership attestation for stores that already exist (designed, not built). |
| +3. **A store's partial first hour** with a charged line holds the month, and nothing releases it. Needs a ruling: an operator release, or the platform absorbs it. |
| +4. **A box's first minutes** before its ledger opened hold that day for everyone. Needs a ruling. |
| + |
| +### Stripe |
| + |
| +5. **No catalog, portal configuration, webhook endpoint or runtime key exists.** The plan and a print-only script are written. Needs scoped approval, then a test-mode rehearsal, then a real purchase. |
| +6. **The quantity worker's fence is unverified against Stripe.** It rests on documented idempotency rules, modelled by a fake. `TestSandboxQuantityFence` must pass in a Stripe sandbox before `CONSOLE_BILLING_QUANTITY_FENCE` is set. Test mode is not authorized, so it has not been run. |
| +7. **The fence is not permanent.** Stripe forgets keys after 24 hours. A sender frozen longer than that can still land an old request. It is detected and held, not prevented. Accepting that, or redesigning how seats are charged, is a decision. |
| +8. **Stripe's treatment of a proration date outside the period** is assumed from documentation. Same sandbox run. |
| +9. **The invoice engine's late-request limit** past Stripe's key memory: detected and held, not prevented; one schedule leaves an open wrong-total invoice the engine will not charge. Whether a customer could pay it from the hosted page is not determined. |
| + |
| +### Decisions that are Paul's |
| + |
| +10. Copy approval for both reviews, and the HALF code word. |
| +11. The proration policy (`CONSOLE_BILLING_PRORATION` stays unset until then). |
| +12. The whole-paid-day rule for partial months. The approved spec only says the allowance is prorated. The rule is tested at its boundaries and is a proposal. |
| +13. Member copy for a bucket retained by another account (`ErrBucketRetained`). |
| + |
| +### Not built |
| + |
| +14. Quantity reduction at renewal. Team plans cannot be sold without it: a renewal bills the old quantity. |
| +15. Free tier: no Free entitlement, no app cap, no storage refusal, no per-plan idle, no flush-only class. |
| +16. Dormancy archive and warnings; recognised-revenue pool accounting; the books site. |
| +17. Per-line tax codes for a month spanning two tax codes (it holds with a review today). |
| +18. A `carlos` verb and documentation for the operator doors; a statement rebuild door for a closed month; an operator verb to backfill bucket ownership. |
| + |
| +### Process |
| + |
| +19. **Round seven review has not passed.** Astra rejected rounds one to six. The lease, fence and simulation work since round six is unreviewed. |
| +20. **The branch tip is not on the forge.** The Amadan hub answered HTTP 500 to every push from 03:30. Bundles are in the run directory. Amadan CI has not run on the tip. |
| + |
| +## Tested on this branch, not proven |
| + |
| +- Bucket-name walks: one at a time across processes, fenced at AWS by signing time. The fence's premise is proven read-only against S3 and IAM, with a list and a get. No delete was sent, and no S3-compatible backend other than AWS was checked. The walks themselves have run only against the in-memory provisioner. Simulation: 30,000 seeds. |
| +- Quantity worker: send lease, key fence, never-reused keys, the hold on every unverified addition. All against a fake provider. Simulation: 500 seeds. |
| +- Partial-month billing, statement folds, the transfer classifier (pinned against the flagship's real usage types), the invoice engine's recovery paths. |
| + |
| +## Proven |
| + |
| +- AWS refuses a request whose signing time is more than fifteen minutes old, on S3 and on IAM, and accepts one thirteen minutes old (read-only calls, 2026-10-01). |
| +- The account's billing from July to September 2026, read-only: no export exists, S3 carried only usage lines, no free transfer allowance reaches the account. |
| +- Nothing else. No purchase, webhook, entitlement write or meter has been verified live. |
diff --git a/implementation/carloku-paid-launch/requirements-audit.json b/implementation/carloku-paid-launch/requirements-audit.json| index 3e6dc8e7d..d9543d7a1 100644 |
| --- a/implementation/carloku-paid-launch/requirements-audit.json |
| +++ b/implementation/carloku-paid-launch/requirements-audit.json |
| @@ -14,14 +14,14 @@ |
| { |
| "step": 2, |
| "requirement": "observed ownership", |
| - "observed_status": "implemented locally; integration review and deployment pending", |
| + "observed_status": "implemented locally; bucket-name walks are serialized across processes by a lease bound to the invocation and fenced at the backend by the request's signing time (checked read-only against S3 and IAM); teardown is a write-ahead record; randomized invariant simulation passes 30,000 seeds; unreviewed since round six; on AWS no ledger can claim completeness until the principal inventory exists; deployment pending", |
| "inspection_anchor": "internal/host/metrics_owner.go; internal/registry/observe.go", |
| "complete": false |
| }, |
| { |
| "step": 3, |
| "requirement": "durable collection and completeness", |
| - "observed_status": "edge proof implemented; durability and store-transfer sources fold into statements with their own completeness (durabilityfold.go, storeegressfold.go) and month evidence; durability days are counted per account so a scoped attestation holds only the accounts it excludes; neither source can read complete in production yet", |
| + "observed_status": "edge proof implemented; durability and store-transfer sources fold into statements with their own completeness (durabilityfold.go, storeegressfold.go) and month evidence; durability days are counted per account; NOTHING ATTESTS: the scoped attestation was reviewed, found unable to prove an exclusive writer, and removed from the branch; both new meters read incomplete in production", |
| "inspection_anchor": "internal/usage/edgespool.go; internal/usage/evidence.go; internal/usage/evidence_month.go", |
| "complete": false |
| }, |
| @@ -35,7 +35,7 @@ |
| { |
| "step": 5, |
| "requirement": "replication writes", |
| - "observed_status": "ledger, hooks and host integration implemented; epoch read race fixed with a -race regression; every unmetered fallback files an escape that makes the box-day incomplete; console fold and billable-write policy implemented and tested; the move worker's checkpoint is tested as metered to the route owner with an escape on the unmetered fallback; scoped attestation chosen (docs/superpowers/plans/2026-10-01-durability-attestation-path.md): per-account fold and scope computation built and tested, the replicator-state reader and the agent call not built, so production attestation remains false", |
| + "observed_status": "ledger, hooks and host integration implemented; epoch read race fixed; every unmetered fallback files an escape; console fold and billable-write policy implemented and tested. LAUNCH BLOCKER: no box can prove its replication writes. Path A (scoped attestation) is parked on feat/carloku-scoped-attestation-20261001 with round-six findings open; path B (the platform's own replicator as sole writer) is designed and not built. Not waived, not scoped out.", |
| "inspection_anchor": "internal/usage/durabilityledger.go; internal/host/durability.go; internal/console/durabilityfold.go", |
| "complete": false |
| }, |
| @@ -77,7 +77,7 @@ |
| { |
| "step": 11, |
| "requirement": "checkout and portal", |
| - "observed_status": "checkout and portal POST doors plus the GET billing page implemented with tests; tailnet demo drives sign-in, step-up, checkout, simulated payment, webhook, active plan and portal against simulated Keymail and Stripe; copy is draft under review; proration policy undecided; no real purchase test", |
| + "observed_status": "checkout and portal POST doors plus the GET billing page implemented with tests; tailnet demo drives sign-in, step-up, checkout, simulated payment, webhook, active plan and portal against simulated Keymail and Stripe; copy is draft under review; proration policy undecided; no real purchase test; the team quantity worker needs two switches, both unset: the proration policy and CONSOLE_BILLING_QUANTITY_FENCE=stripe-sandbox-verified; its provider-side fence rests on Stripe's documented idempotency rules, is modelled by a fake in tests, is NOT verified against Stripe, and is not a permanent token (keys are forgotten after 24 hours)", |
| "inspection_anchor": "internal/console/billinghandlers.go; internal/console/billingpage.go; internal/console/billingflow_demo_test.go", |
| "complete": false |
| }, |
| @@ -146,8 +146,11 @@ |
| "scoped new-infrastructure approval", |
| "deployment version verification", |
| "actual Stripe purchase/entitlement/portal proof", |
| - "approved public copy and deployed tax configuration" |
| + "approved public copy and deployed tax configuration", |
| + "replication-write completeness proven on a box (path B built, reviewed, approved, deployed)", |
| + "Stripe sandbox verification of the quantity fence (TestSandboxQuantityFence) before the worker is enabled", |
| + "round seven independent review (Astra and Opus) passing on the exact tip" |
| ], |
| "audited_at": "2026-10-01", |
| - "branch_state": "pushed to origin feat/carloku-paid-launch-20260916; not merged; billing off; nothing deployed" |
| + "branch_state": "local tip ahead of origin (the Amadan hub answered HTTP 500 to pushes from 03:30 on 2026-10-01; bundles are in the run directory); not merged; billing off; quantity worker off; nothing deployed" |
| } |
diff --git a/implementation/carloku-paid-launch/stripe-provisioning.md b/implementation/carloku-paid-launch/stripe-provisioning.md| index d1c257a76..ad67c8fc8 100644 |
| --- a/implementation/carloku-paid-launch/stripe-provisioning.md |
| +++ b/implementation/carloku-paid-launch/stripe-provisioning.md |
| @@ -282,10 +282,15 @@ Deliberately not granted: |
| - Subscription items write, for now. The team quantity worker |
| (`internal/console/billingquantity.go`) calls |
| - `POST /v1/subscription_items/{id}`, but only once `CONSOLE_BILLING_PRORATION` |
| - is set, and that policy is undecided. Grant this permission in the same |
| - change that sets the variable, not before: until then the key cannot raise a |
| - quantity even by mistake. |
| + `POST /v1/subscription_items/{id}`, to change a quantity and, with a |
| + metadata-only body, to fence an operation's idempotency key. It runs only |
| + once two variables are set: `CONSOLE_BILLING_PRORATION` (the policy, which |
| + is undecided) and `CONSOLE_BILLING_QUANTITY_FENCE=stripe-sandbox-verified` |
| + (set only after `TestSandboxQuantityFence` has passed against a Stripe |
| + test-mode account; see |
| + `docs/superpowers/specs/2026-10-01-stripe-idempotency-evidence.md`). Grant |
| + this permission in the same change that sets both, not before: until then |
| + the key cannot raise a quantity even by mistake. |
| - Events. Nothing calls `/v1/events`. The webhook handler authenticates by |
| signature and reconciles by reading the subscription. |
| - Promotion codes. Checkout resolves `allow_promotion_codes` on Stripe's side. |
diff --git a/internal/billingstripe/sandbox_fence_test.go b/internal/billingstripe/sandbox_fence_test.go| new file mode 100644 |
| index 000000000..522823452 |
| --- /dev/null |
| +++ b/internal/billingstripe/sandbox_fence_test.go |
| @@ -0,0 +1,292 @@ |
| +package billingstripe |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "io" |
| + "net/http" |
| + "net/url" |
| + "os" |
| + "strconv" |
| + "strings" |
| + "sync" |
| + "testing" |
| + "time" |
| +) |
| + |
| +// TestSandboxQuantityFence checks, against Stripe itself, the provider |
| +// behaviour the team quantity worker's fence rests on |
| +// (internal/console/billingquantity.go, "The fence"). Everything else that |
| +// tests the fence runs against a fake that models Stripe's documented |
| +// rules. That proves the console's logic given the rules. This is what |
| +// proves the rules. |
| +// |
| +// It is the gate on the worker: CONSOLE_BILLING_QUANTITY_FENCE is set to |
| +// "stripe-sandbox-verified" only after this has passed, and the log of the |
| +// passing run is kept with the launch evidence. |
| +// |
| +// It runs only when CARLOS_STRIPE_SANDBOX_KEY holds a TEST-MODE restricted |
| +// key (rk_test_...) with write access to customers, products, prices and |
| +// subscriptions and read access to invoices. It refuses any other key. It |
| +// creates one customer, product, price and subscription in the sandbox, |
| +// and cancels and deletes them when it ends. It never prints the key. |
| +// |
| +// What it cannot check: what happens after the provider has forgotten a |
| +// key. Stripe documents that keys may be removed once they are at least 24 |
| +// hours old and that a key reused after that runs as a new request. The |
| +// fence is therefore not permanent, the console treats it as bounded, and |
| +// no run of this test changes that. |
| +func TestSandboxQuantityFence(t *testing.T) { |
| + key := os.Getenv("CARLOS_STRIPE_SANDBOX_KEY") |
| + if key == "" { |
| + t.Skip("set CARLOS_STRIPE_SANDBOX_KEY to a Stripe test-mode restricted key (rk_test_...) to verify the quantity fence against Stripe") |
| + } |
| + if !strings.HasPrefix(key, "rk_test_") { |
| + t.Fatal("CARLOS_STRIPE_SANDBOX_KEY is not a test-mode restricted key (rk_test_...); refusing to run") |
| + } |
| + ctx := context.Background() |
| + c, err := NewClient(key) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + httpc := &http.Client{Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} |
| + type answer struct { |
| + status int |
| + body map[string]any |
| + replayed bool |
| + } |
| + // raw sends one request and returns what Stripe answered. Only the |
| + // status, the error's type, code and param, and named fields are ever |
| + // logged: never a body, never the key. |
| + raw := func(method, path string, form url.Values, idem string) answer { |
| + t.Helper() |
| + var body io.Reader |
| + if method == http.MethodPost { |
| + body = strings.NewReader(form.Encode()) |
| + } else if len(form) > 0 { |
| + path += "?" + form.Encode() |
| + } |
| + req, err := http.NewRequestWithContext(ctx, method, defaultBaseURL+path, body) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + req.Header.Set("Authorization", "Bearer "+key) |
| + req.Header.Set("Stripe-Version", apiVersion) |
| + if method == http.MethodPost { |
| + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") |
| + if idem != "" { |
| + req.Header.Set("Idempotency-Key", idem) |
| + } |
| + } |
| + resp, err := httpc.Do(req) |
| + if err != nil { |
| + t.Fatalf("%s %s: request failed (no body shown)", method, strings.SplitN(path, "?", 2)[0]) |
| + } |
| + defer resp.Body.Close() |
| + out := answer{status: resp.StatusCode, replayed: resp.Header.Get("Idempotent-Replayed") == "true"} |
| + rawBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) |
| + _ = json.Unmarshal(rawBody, &out.body) |
| + return out |
| + } |
| + errOf := func(a answer) string { |
| + e, _ := a.body["error"].(map[string]any) |
| + return fmt.Sprintf("status=%d type=%v code=%v param=%v", a.status, e["type"], e["code"], e["param"]) |
| + } |
| + id := func(a answer, what string) string { |
| + t.Helper() |
| + s, _ := a.body["id"].(string) |
| + if a.status != 200 || s == "" { |
| + t.Fatalf("setup: create %s: %s", what, errOf(a)) |
| + } |
| + return s |
| + } |
| + run := strconv.FormatInt(time.Now().UnixNano(), 36) |
| + k := func(name string) string { return "carloku-fence-check-" + run + "-" + name } |
| + |
| + // Setup: one annual subscription, quantity 10, invoiced (no card). |
| + customer := raw("POST", "/v1/customers", url.Values{"name": {"carloku fence check " + run}, "metadata[carloku_fence_check]": {run}}, k("customer")) |
| + customerID := id(customer, "customer") |
| + if live, _ := customer.body["livemode"].(bool); live { |
| + t.Fatal("the key is live-mode; refusing to continue") |
| + } |
| + t.Cleanup(func() { raw("DELETE", "/v1/customers/"+customerID, nil, "") }) |
| + productID := id(raw("POST", "/v1/products", url.Values{"name": {"carloku fence check " + run}}, k("product")), "product") |
| + t.Cleanup(func() { raw("POST", "/v1/products/"+productID, url.Values{"active": {"false"}}, k("product-off")) }) |
| + priceID := id(raw("POST", "/v1/prices", url.Values{"product": {productID}, "currency": {"eur"}, "unit_amount": {"1000"}, "recurring[interval]": {"year"}}, k("price")), "price") |
| + subID := id(raw("POST", "/v1/subscriptions", url.Values{"customer": {customerID}, "items[0][price]": {priceID}, "items[0][quantity]": {"10"}, |
| + "collection_method": {"send_invoice"}, "days_until_due": {"30"}}, k("subscription")), "subscription") |
| + canceled := false |
| + t.Cleanup(func() { |
| + if !canceled { |
| + raw("DELETE", "/v1/subscriptions/"+subID, nil, "") |
| + } |
| + }) |
| + sub, err := c.GetSubscription(ctx, subID) |
| + if err != nil || len(sub.Items) != 1 || sub.Items[0].Quantity != 10 { |
| + t.Fatalf("setup: read the subscription: %v", err) |
| + } |
| + item := sub.Items[0].ID |
| + at := sub.CurrentPeriodStart.Add(time.Minute) |
| + quantity := func() int64 { |
| + t.Helper() |
| + s, err := c.GetSubscription(ctx, subID) |
| + if err != nil || len(s.Items) != 1 { |
| + t.Fatalf("read the subscription: %v", err) |
| + } |
| + return s.Items[0].Quantity |
| + } |
| + count := func(path string) int { |
| + t.Helper() |
| + a := raw("GET", path, url.Values{"customer": {customerID}, "limit": {"100"}}, "") |
| + data, _ := a.body["data"].([]any) |
| + if a.status != 200 { |
| + t.Fatalf("list %s: %s", path, errOf(a)) |
| + } |
| + return len(data) |
| + } |
| + update := func(target int64, date time.Time, idem string) error { |
| + _, err := c.UpdateQuantityWith(ctx, item, target, ProrationCreateProrations, date, idem) |
| + return err |
| + } |
| + fenceForm := url.Values{"metadata[carloku_quantity_fence]": {"1"}} |
| + |
| + t.Run("1 a fence taken first refuses the quantity update that arrives after it", func(t *testing.T) { |
| + items, invoices := count("/v1/invoiceitems"), count("/v1/invoices") |
| + a := raw("POST", "/v1/subscription_items/"+item, fenceForm, k("one")) |
| + if a.status != 200 { |
| + t.Fatalf("the fence request was not accepted: %s", errOf(a)) |
| + } |
| + if q := quantity(); q != 10 { |
| + t.Fatalf("the fence request changed the quantity to %d", q) |
| + } |
| + if i, v := count("/v1/invoiceitems"), count("/v1/invoices"); i != items || v != invoices { |
| + t.Fatalf("the fence request created billing objects: invoice items %d -> %d, invoices %d -> %d", items, i, invoices, v) |
| + } |
| + err := update(11, at, k("one")) |
| + var refused *Error |
| + if !errors.As(err, &refused) || refused.HTTPStatus != 400 || refused.StripeType != "idempotency_error" { |
| + t.Fatalf("the late quantity update under the fenced key: %v, want a 400 idempotency_error", err) |
| + } |
| + if q := quantity(); q != 10 { |
| + t.Fatalf("the late quantity update ran: quantity %d", q) |
| + } |
| + if fenced, err := c.FenceQuantityKey(ctx, item, 1, k("one")); err != nil || !fenced { |
| + t.Fatalf("FenceQuantityKey on a key it already holds: fenced=%v err=%v", fenced, err) |
| + } |
| + t.Logf("VERIFIED: fence 200, quantity and billing objects unchanged; late update refused with 400 idempotency_error") |
| + }) |
| + |
| + t.Run("2 a fence taken after the update ran finds the key taken, and a second copy of the update runs nothing", func(t *testing.T) { |
| + if err := update(11, at, k("two")); err != nil { |
| + t.Fatalf("the quantity update: %v", err) |
| + } |
| + items := count("/v1/invoiceitems") |
| + form := url.Values{"quantity": {"11"}, "proration_behavior": {ProrationCreateProrations}, "proration_date": {strconv.FormatInt(at.Unix(), 10)}} |
| + again := raw("POST", "/v1/subscription_items/"+item, form, k("two")) |
| + if again.status != 200 || !again.replayed { |
| + t.Fatalf("the second copy of the update: %s replayed=%v, want 200 answered from memory (Idempotent-Replayed: true)", errOf(again), again.replayed) |
| + } |
| + if i := count("/v1/invoiceitems"); i != items { |
| + t.Fatalf("the second copy of the update ran: invoice items %d -> %d", items, i) |
| + } |
| + a := raw("POST", "/v1/subscription_items/"+item, fenceForm, k("two")) |
| + if e, _ := a.body["error"].(map[string]any); a.status != 400 || e["type"] != "idempotency_error" { |
| + t.Fatalf("the fence under a key the update holds: %s, want 400 idempotency_error", errOf(a)) |
| + } |
| + if fenced, err := c.FenceQuantityKey(ctx, item, 2, k("two")); err != nil || !fenced { |
| + t.Fatalf("FenceQuantityKey under a key the update holds: fenced=%v err=%v", fenced, err) |
| + } |
| + if q := quantity(); q != 11 { |
| + t.Fatalf("quantity %d, want 11", q) |
| + } |
| + t.Logf("VERIFIED: replay answered from memory with no new proration; fence under the update's key refused with 400 idempotency_error") |
| + }) |
| + |
| + t.Run("3 a request refused as invalid does not take its key", func(t *testing.T) { |
| + err := update(12, sub.CurrentPeriodStart.Add(-48*time.Hour), k("three")) |
| + var refused *Error |
| + if !errors.As(err, &refused) || !refused.RefusedParam("proration_date") { |
| + t.Fatalf("an update dated before the period: %v, want a 400 invalid_request_error naming proration_date", err) |
| + } |
| + if q := quantity(); q != 11 { |
| + t.Fatalf("the refused update ran: quantity %d", q) |
| + } |
| + a := raw("POST", "/v1/subscription_items/"+item, fenceForm, k("three")) |
| + if a.status != 200 || a.replayed { |
| + t.Fatalf("the fence under the refused request's key: %s replayed=%v, want a fresh 200 (the refusal saved nothing)", errOf(a), a.replayed) |
| + } |
| + err = update(12, at, k("three")) |
| + if !errors.As(err, &refused) || refused.HTTPStatus != 400 || refused.StripeType != "idempotency_error" { |
| + t.Fatalf("the update under the now fenced key: %v, want a 400 idempotency_error", err) |
| + } |
| + if q := quantity(); q != 11 { |
| + t.Fatalf("quantity %d, want 11", q) |
| + } |
| + t.Logf("VERIFIED: a proration date outside the period is refused naming proration_date and saves no key; the fence then takes the key") |
| + }) |
| + |
| + t.Run("4 concurrent copies of one request run it once", func(t *testing.T) { |
| + const copies = 6 |
| + items := count("/v1/invoiceitems") |
| + form := url.Values{"quantity": {"12"}, "proration_behavior": {ProrationCreateProrations}, "proration_date": {strconv.FormatInt(at.Unix(), 10)}} |
| + answers := make([]answer, copies) |
| + var wg sync.WaitGroup |
| + for i := range answers { |
| + wg.Add(1) |
| + go func() { |
| + defer wg.Done() |
| + answers[i] = raw("POST", "/v1/subscription_items/"+item, form, k("four")) |
| + }() |
| + } |
| + wg.Wait() |
| + ok, conflict := 0, 0 |
| + for _, a := range answers { |
| + switch a.status { |
| + case 200: |
| + ok++ |
| + case 409: |
| + conflict++ |
| + default: |
| + t.Fatalf("a concurrent copy was answered %s, want 200 or 409", errOf(a)) |
| + } |
| + } |
| + if ok == 0 || quantity() != 12 { |
| + t.Fatalf("no copy ran: %d ok, %d conflict", ok, conflict) |
| + } |
| + if i := count("/v1/invoiceitems"); i-items > 2 { |
| + t.Fatalf("the update ran more than once: invoice items %d -> %d", items, i) |
| + } |
| + if conflict == 0 { |
| + t.Logf("NOT OBSERVED: no copy was answered 409 in this run (%d answered 200); the in-use answer is documented, and FenceQuantityKey treats 409 as not fenced", ok) |
| + } else { |
| + t.Logf("VERIFIED: %d copies answered 200, %d answered 409; the update ran once", ok, conflict) |
| + } |
| + }) |
| + |
| + t.Run("5 a canceled subscription", func(t *testing.T) { |
| + if a := raw("DELETE", "/v1/subscriptions/"+subID, nil, ""); a.status != 200 { |
| + t.Fatalf("cancel: %s", errOf(a)) |
| + } |
| + canceled = true |
| + err := update(13, at, k("five-update")) |
| + if err == nil { |
| + t.Fatal("a quantity update on a canceled subscription's item was accepted") |
| + } |
| + t.Logf("VERIFIED: a quantity update on a canceled subscription's item is refused (%v)", err) |
| + fenced, ferr := c.FenceQuantityKey(ctx, item, 5, k("five-fence")) |
| + switch { |
| + case ferr == nil && fenced: |
| + t.Logf("VERIFIED: the fence on a canceled subscription's item is taken") |
| + default: |
| + // The worker would file billing_quantity_fence_refused and |
| + // hold the account here. That is safe and it is not what |
| + // anyone wants for every canceled subscription: it needs a |
| + // code decision before the worker is switched on. |
| + t.Fatalf("the fence on a canceled subscription's item: fenced=%v err=%v; the worker's ended-subscription path needs a decision", fenced, ferr) |
| + } |
| + }) |
| + |
| + t.Log("NOT VERIFIABLE HERE: behaviour after the provider has forgotten a key (documented: at least 24 hours, then a reused key runs as a new request). The fence is bounded by that, by design.") |
| +} |
diff --git a/internal/console/billingquantity.go b/internal/console/billingquantity.go| index c6ecce2e6..4a8cbeb19 100644 |
| --- a/internal/console/billingquantity.go |
| +++ b/internal/console/billingquantity.go |
| @@ -74,6 +74,9 @@ import ( |
| const ( |
| billingProrationImmediate = "immediate" |
| billingProrationNextInvoice = "next_invoice" |
| + // billingQuantityFenceVerified is the only value of |
| + // BillingConfig.QuantityFence that lets the worker run. |
| + billingQuantityFenceVerified = "stripe-sandbox-verified" |
| quantityOutcomeCharged = "charged" |
| quantityOutcomeIncluded = "included" |
| @@ -86,6 +89,9 @@ const ( |
| // operation's key. The operation cannot be acknowledged without the |
| // fence, so the account's quantity work waits on a person. |
| billingQuantityFenceReason = "billing_quantity_fence_refused" |
| + // Filed when the provider's quantity is observed to have gone down |
| + // inside a period. The console never asks for that. |
| + billingQuantityLoweredReason = "billing_quantity_lowered" |
| ) |
| // prorationBehavior maps the configured policy to the provider's behaviour |
| @@ -101,6 +107,19 @@ func (cfg BillingConfig) prorationBehavior() (behavior, outcome string, ok bool) |
| return "", "", false |
| } |
| +// quantityWorker reports whether the quantity worker may run, and under |
| +// which behaviour. It needs both switches: a decided proration policy, and |
| +// the record that the provider's key semantics the fence rests on were |
| +// verified in a sandbox. Either one missing leaves additions pending, |
| +// exactly as an undecided policy does. |
| +func (cfg BillingConfig) quantityWorker() (behavior string, ok bool) { |
| + behavior, _, ok = cfg.prorationBehavior() |
| + if !ok || cfg.QuantityFence != billingQuantityFenceVerified { |
| + return "", false |
| + } |
| + return behavior, true |
| +} |
| + |
| // pendingQuantityAccounts lists, in account order, every account the pass |
| // has work for: an unprocessed addition bound to a subscription, or an |
| // operation left unacknowledged (a crash between the mark and the |
| @@ -589,12 +608,30 @@ func (s *Store) retireQuantityOperation(accountID int64, key string, now time.Ti |
| // provider read taken after the fence, never before it, so a request that |
| // ran just before the fence is seen and recorded, not retired as unsent. |
| // |
| -// Its limit: the provider remembers a key for a bounded time (Stripe |
| -// documents at least 24 hours). A request delayed longer than that after |
| -// the fence is outside what a key can stop. What remains then is the |
| -// request's own date, which the provider refuses outside the period it was |
| -// computed for, and the mismatch review, which holds the account when the |
| -// provider's quantity is neither what the console bills nor what it wants. |
| +// Its limit. A consumed key is NOT a permanent fencing token. Stripe |
| +// documents that keys may be removed once they are at least 24 hours old, |
| +// and that "we generate a new request if a key is reused after the original |
| +// is pruned". A request that first reaches the provider after that (its |
| +// sender frozen for more than a day between its last check and the wire) |
| +// runs as new, and nothing in this design refuses it. Two things remain: |
| +// |
| +// - the request's own date: the provider refuses a proration date |
| +// outside the subscription's current period, so it cannot be charged |
| +// into a later period; |
| +// - detection, not prevention: such a request can only be for an older, |
| +// lower target, and a quantity that goes down inside a period files |
| +// billing_quantity_lowered (billingservice.go), which holds the |
| +// account's invoices for a person. |
| +// |
| +// What is proven and what is not. Everything above about the provider is |
| +// from Stripe's documentation (docs/superpowers/specs/ |
| +// 2026-10-01-stripe-idempotency-evidence.md quotes it). The tests here run |
| +// against a fake that models those documented rules; they prove this |
| +// code's logic given the rules, not the rules. The rules are checked |
| +// against Stripe itself by TestSandboxQuantityFence in |
| +// internal/billingstripe, which runs only with a test-mode key and has not |
| +// been run. Until it has passed, the worker must stay off: it starts only |
| +// when CONSOLE_BILLING_QUANTITY_FENCE says so (BillingConfig.quantityWorker). |
| const quantityOperationSettled = `(confirmed_at<>0 OR fenced_at<>0)` |
| // markQuantityFenced records that the provider holds the operation's key. |
| @@ -669,7 +706,7 @@ func (s *Server) billingQuantityPass(ctx context.Context, now time.Time) error { |
| if b == nil { |
| return nil |
| } |
| - behavior, _, ok := b.cfg.prorationBehavior() |
| + behavior, ok := b.cfg.quantityWorker() |
| if !ok { |
| return nil |
| } |
diff --git a/internal/console/billingquantity_test.go b/internal/console/billingquantity_test.go| index 4de18695c..3f5c9ef8b 100644 |
| --- a/internal/console/billingquantity_test.go |
| +++ b/internal/console/billingquantity_test.go |
| @@ -177,7 +177,7 @@ func newQuantityFixture(t *testing.T, proration string) (*consoleFixture, *quant |
| t.Fatal(err) |
| } |
| fx.srv.billing = &billingRuntime{locks: newStoreWalkLocks(), client: client, cfg: BillingConfig{ |
| - MerchantAccount: "acct_carloku", TaxCode: "txcd_test", PortalConfiguration: "bpc_safe", LiveMode: true, Proration: proration, |
| + MerchantAccount: "acct_carloku", TaxCode: "txcd_test", PortalConfiguration: "bpc_safe", LiveMode: true, Proration: proration, QuantityFence: billingQuantityFenceVerified, |
| CountryBands: map[string]string{"IE": "A"}, WebhookSecrets: []string{"whsec_test"}, |
| Prices: map[string]string{billingPriceKey(plans.CurrentTermsVersion, plans.PlanCommunity, "A"): "price_community_A"}, |
| }} |
| @@ -284,6 +284,26 @@ func TestQuantityWorkerRaisesTheSubscriptionAndClearsTheHold(t *testing.T) { |
| } |
| } |
| +// A decided policy does not start the worker by itself. Its fence rests on |
| +// provider behaviour that has to be verified in a sandbox first, and until |
| +// an operator records that it was, the worker does exactly what it does |
| +// with no policy: nothing reaches the provider and the additions wait. |
| +func TestQuantityWorkerDoesNothingUntilTheFenceIsVerified(t *testing.T) { |
| + for _, fence := range []string{"", "verified", "yes", "stripe-sandbox"} { |
| + fx, q, now := newQuantityFixture(t, billingProrationImmediate) |
| + fx.srv.billing.cfg.QuantityFence = fence |
| + if err := fx.st.AddMember(OpsAccountID, "eleventh@example.test", RoleMember, now.Add(time.Minute)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := fx.srv.billingQuantityPass(context.Background(), now.Add(2*time.Minute)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if q.gets != 0 || len(q.posts) != 0 || len(q.fences) != 0 || len(pendingAdditions(t, fx)) != 1 || !quantityHeld(t, fx) { |
| + t.Fatalf("fence %q: the worker acted before the fence was verified: gets=%d posts=%v fences=%v", fence, q.gets, q.posts, q.fences) |
| + } |
| + } |
| +} |
| + |
| // No policy, no provider call, no mark: the decision is not made for Paul. |
| func TestQuantityWorkerDoesNothingWithoutAPolicy(t *testing.T) { |
| fx, q, now := newQuantityFixture(t, "") |
| @@ -2058,3 +2078,42 @@ func TestBeginQuantityOperationNeverReusesAKey(t *testing.T) { |
| t.Fatalf("a fresh key was refused: started=%v err=%v", started, err) |
| } |
| } |
| + |
| +// The console never lowers a quantity inside a period. When the provider's |
| +// quantity is observed to have gone down, by a hand or by a request the |
| +// provider could no longer refuse, the number is recorded (it is what the |
| +// customer is charged for now) and a review is filed: it is never taken |
| +// quietly. A quantity that goes up by a hand files nothing here. |
| +func TestQuantityLoweredAtTheProviderFilesAReview(t *testing.T) { |
| + fx, q, now := newQuantityFixture(t, billingProrationImmediate) |
| + ctx := context.Background() |
| + if err := fx.st.AddMember(OpsAccountID, "eleventh@example.test", RoleMember, now.Add(time.Minute)); err != nil { |
| + t.Fatal(err) |
| + } |
| + for i := 0; i < 2; i++ { |
| + if err := fx.srv.billingQuantityPass(ctx, time.Now().UTC()); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if state, _ := fx.st.billingAccount(OpsAccountID); state.Quantity != 11 { |
| + t.Fatalf("fixture: billed %d, want 11", state.Quantity) |
| + } |
| + if rev, err := fx.st.billingReview(OpsAccountID, "sub_team", billingQuantityLoweredReason); err == nil && rev.Occurrences > 0 { |
| + t.Fatalf("a raise filed the lowering review: %+v", rev) |
| + } |
| + q.mu.Lock() |
| + q.quantity = 10 |
| + q.mu.Unlock() |
| + for i := 0; i < 2; i++ { |
| + if err := fx.srv.reconcileBillingSubscription(ctx, "sub_team", fmt.Sprintf("evt_lowered_%d", i), time.Now().UTC()); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if state, _ := fx.st.billingAccount(OpsAccountID); state.Quantity != 10 { |
| + t.Fatalf("billed %d: the provider's quantity is what the customer is charged for", state.Quantity) |
| + } |
| + rev, err := fx.st.billingReview(OpsAccountID, "sub_team", billingQuantityLoweredReason) |
| + if err != nil || rev.ResolvedAt != 0 || rev.Occurrences != 1 { |
| + t.Fatalf("a quantity lowered inside the period was taken with no review, or filed twice: %+v %v", rev, err) |
| + } |
| +} |
diff --git a/internal/console/billingservice.go b/internal/console/billingservice.go| index 14bc6a5f7..98818dff0 100644 |
| --- a/internal/console/billingservice.go |
| +++ b/internal/console/billingservice.go |
| @@ -33,6 +33,17 @@ type BillingConfig struct { |
| // invoice. Empty means the policy is undecided: nothing is sent to the |
| // provider and the additions stay pending, holding the monthly freeze. |
| Proration string |
| + // QuantityFence is the second switch the quantity worker needs, and |
| + // the explicit gate on its one unproven dependency. The worker's |
| + // safety against a request that arrives late rests on how the provider |
| + // treats idempotency keys (billingquantity.go, "The fence"). That has |
| + // been read in the provider's documentation and modelled in tests, and |
| + // not yet run against the provider. The worker starts only when this |
| + // is "stripe-sandbox-verified", which an operator sets after |
| + // TestSandboxQuantityFence (internal/billingstripe) has passed against |
| + // a Stripe test-mode account. A decided proration policy alone does |
| + // not start it. |
| + QuantityFence string |
| } |
| type billingRuntime struct { |
| @@ -80,6 +91,9 @@ func newBillingRuntime(cfg *BillingConfig) (*billingRuntime, error) { |
| if cfg.Proration != "" && cfg.Proration != billingProrationImmediate && cfg.Proration != billingProrationNextInvoice { |
| return nil, errBillingUnavailable |
| } |
| + if cfg.QuantityFence != "" && cfg.QuantityFence != billingQuantityFenceVerified { |
| + return nil, errBillingUnavailable |
| + } |
| for country, band := range cfg.CountryBands { |
| if len(country) != 2 || strings.ToUpper(country) != country { |
| return nil, errBillingUnavailable |
| @@ -487,6 +501,20 @@ func (s *Server) reconcileBillingSubscription(ctx context.Context, id, eventID s |
| } |
| renewalStart = sub.CurrentPeriodStart |
| } |
| + // A quantity that went down inside a period was not lowered by this |
| + // console: it only ever raises one (a member who leaves comes off at |
| + // renewal). So it was a hand at the provider, or a request of an |
| + // operation long since retired that arrived after the provider had |
| + // forgotten its key, which nothing can refuse (billingquantity.go, the |
| + // fence's limit). The provider's number is still recorded, because it |
| + // is what the customer is now charged for. But it is never recorded |
| + // quietly: a review is filed, which holds the account's invoices until |
| + // a person has looked. |
| + if state.Subscription == id && state.PeriodStart == next.PeriodStart && next.Quantity < state.Quantity { |
| + if err := s.store.fileBillingReviewOnce(acct.ID, id, billingQuantityLoweredReason, now); err != nil { |
| + return err |
| + } |
| + } |
| // A quantity change that is the account's own outstanding operation |
| // landing takes effect at the operation's instant, the proration date |
| // the provider was given, whichever path observes it first (the worker's |
diff --git a/internal/objstore/objstore.go b/internal/objstore/objstore.go| index 63a8656ef..e1468a6ea 100644 |
| --- a/internal/objstore/objstore.go |
| +++ b/internal/objstore/objstore.go |
| @@ -155,7 +155,10 @@ type TeardownBlock struct { |
| StartedAt time.Time `json:"started_at"` |
| // BucketDeletedAt: this store's bucket was emptied and deleted. A |
| // re-run must never empty or delete a bucket of that name again: if |
| - // one exists now, somebody else created it. |
| + // one exists now, somebody else created it. That holds only until |
| + // this store is granted again: a grant over the unfinished teardown |
| + // may create the bucket itself, so it voids this note before it does |
| + // anything else, and a later teardown decides again from the ledger. |
| BucketDeletedAt time.Time `json:"bucket_deleted_at,omitzero"` |
| } |
diff --git a/internal/objstore/ownership_test.go b/internal/objstore/ownership_test.go| index dea2ef67a..6a2e2c5a5 100644 |
| --- a/internal/objstore/ownership_test.go |
| +++ b/internal/objstore/ownership_test.go |
| @@ -1569,3 +1569,64 @@ func TestTeardownStopsWhenTheLedgerNoLongerCarriesItsDecision(t *testing.T) { |
| } |
| } |
| } |
| + |
| +// A teardown deletes the bucket and notes that on the record, then fails |
| +// before it finishes. A grant takes the store back, creates the bucket |
| +// again, and fails too. The teardown is run again. Its old note says the |
| +// bucket is deleted, and that is no longer true: the grant voided the note |
| +// when it began, so the teardown decides again and deletes the bucket the |
| +// grant brought back. Nothing is left that nobody holds. |
| +func TestTeardownAfterAFailedTakeBackDeletesTheBucketTheGrantRecreated(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + _, pub := testSealKeypair(t) |
| + p := NewMemProvisioner("eu-west-1") |
| + advance := leaseClock(t) |
| + declare(t, s, "bab", "titogo", "TITO_S3") |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", t0); err != nil { |
| + t.Fatal(err) |
| + } |
| + p.FailNext = map[string]error{"DeleteUser": errors.New("injected DeleteUser")} |
| + if err := Teardown(ctx, s, p, nil, "bab", "titogo"); err == nil { |
| + t.Fatal("fixture: the teardown was meant to fail after deleting the bucket") |
| + } |
| + if _, exists := p.Buckets[ledgerBucket]; exists { |
| + t.Fatal("fixture: the first teardown did not delete the bucket") |
| + } |
| + rec, _, err := Get(ctx, s, "bab", "titogo") |
| + if err != nil || rec == nil || rec.Teardown == nil || rec.Teardown.BucketDeletedAt.IsZero() { |
| + t.Fatalf("fixture: the record does not note the deleted bucket: %+v %v", rec, err) |
| + } |
| + advance(time.Hour) |
| + p.FailNext = map[string]error{"MintKey": errors.New("injected MintKey")} |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", t0.Add(time.Hour)); err == nil { |
| + t.Fatal("fixture: the grant was meant to fail after creating the bucket") |
| + } |
| + if _, exists := p.Buckets[ledgerBucket]; !exists { |
| + t.Fatal("fixture: the failed grant did not recreate the bucket") |
| + } |
| + rec, _, _ = Get(ctx, s, "bab", "titogo") |
| + if rec == nil || rec.Teardown == nil || !rec.Teardown.BucketDeletedAt.IsZero() { |
| + t.Fatalf("the grant left the teardown's note that the bucket is deleted: %+v", rec) |
| + } |
| + advance(time.Hour) |
| + p.FailNext = nil |
| + if err := Teardown(ctx, s, p, nil, "bab", "titogo"); err != nil { |
| + t.Fatalf("the teardown run again: %v", err) |
| + } |
| + if _, exists := p.Buckets[ledgerBucket]; exists { |
| + t.Fatal("the bucket the grant recreated was left with nobody holding it") |
| + } |
| + if _, ok := p.Users[UserName("bab", "titogo")]; ok { |
| + t.Fatal("the user was left") |
| + } |
| + ledger, _, err := ReadBucketOwnership(ctx, s, ledgerBucket) |
| + if err != nil || ledger == nil || ledger.Destroying != nil { |
| + t.Fatalf("ledger after the teardown: %+v %v", ledger, err) |
| + } |
| + for _, iv := range ledger.Intervals { |
| + if iv.open() { |
| + t.Fatalf("an interval is still open: %+v", iv) |
| + } |
| + } |
| +} |
diff --git a/internal/objstore/walk.go b/internal/objstore/walk.go| index f764d857b..0c9631d13 100644 |
| --- a/internal/objstore/walk.go |
| +++ b/internal/objstore/walk.go |
| @@ -210,6 +210,21 @@ func Grant(ctx context.Context, s release.Store, p Provisioner, d Deployment, |
| wroteGrant := rec.Grant == nil |
| if wroteGrant { |
| rec.Grant = &GrantBlock{Backend: d.Backend, Bucket: bucket, Source: source, At: now, By: actor, Minted: adoptBucket == ""} |
| + } |
| + // A grant over an unfinished teardown voids what that teardown noted |
| + // about the bucket, before this walk does anything that could bring |
| + // the bucket back. The note "the bucket was deleted" is true only |
| + // until a walk entitled to create it begins. Left standing, a grant |
| + // that recreated the bucket and then failed would hand the resumed |
| + // teardown a note saying there is nothing to delete, and the bucket |
| + // would be left with nobody holding it (simulation seed 1085). With |
| + // the note gone the teardown decides again from the ledger, and a |
| + // bucket that is not there costs it nothing. |
| + voided := rec.Teardown != nil && !rec.Teardown.BucketDeletedAt.IsZero() |
| + if voided { |
| + rec.Teardown.BucketDeletedAt = time.Time{} |
| + } |
| + if wroteGrant || voided { |
| if etag, err = Put(ctx, s, acct, app, rec, etag); err != nil { |
| return nil, err |
| } |
diff --git a/internal/objstore/walk_sim_test.go b/internal/objstore/walk_sim_test.go| index a897db940..5ee5ec522 100644 |
| --- a/internal/objstore/walk_sim_test.go |
| +++ b/internal/objstore/walk_sim_test.go |
| @@ -2,10 +2,12 @@ package objstore |
| import ( |
| "context" |
| + "encoding/json" |
| "errors" |
| "fmt" |
| "math/rand" |
| "os" |
| + "sort" |
| "strconv" |
| "strings" |
| "testing" |
| @@ -86,7 +88,12 @@ func TestBucketNameInvariantsUnderRandomWalks(t *testing.T) { |
| if testing.Short() { |
| seeds = 60 |
| } |
| - for seed := 0; seed < seeds; seed++ { |
| + first := 0 |
| + // CARLOS_SIM_SEED runs one seed, to reproduce a failure. |
| + if n, err := strconv.Atoi(os.Getenv("CARLOS_SIM_SEED")); err == nil && n >= 0 { |
| + first, seeds = n, n+1 |
| + } |
| + for seed := first; seed < seeds; seed++ { |
| if trace, problem := simulateBucketName(t, int64(seed), 40); problem != "" { |
| t.Fatalf("seed %d: %s\n%s", seed, problem, strings.Join(trace, "\n")) |
| } |
| @@ -160,10 +167,24 @@ func simulateBucketName(t *testing.T, seed int64, steps int) (trace []string, pr |
| return nil |
| } |
| // Asked to go: from this instant the store is not owed its data. |
| + was := a.state |
| a.state = "broken" |
| estate := rec.Grant != nil || rec.Provisioned != nil |
| err := Teardown(ctx, s, p, sc, a.acct, a.app) |
| logf("teardown %s: %v", a.name(), err) |
| + if errors.Is(err, ErrBucketBusy) { |
| + // Refused before it began: another walk holds the name. The |
| + // walk made no call and wrote nothing, so the store is what it |
| + // was. (Seed 778 found the model forgetting this.) |
| + a.state = was |
| + return err |
| + } |
| + if err != nil && was == "retained" { |
| + // A retained bucket was reported to a person. A later attempt |
| + // that fails does not withdraw that report. |
| + a.state = "retained" |
| + return err |
| + } |
| if errors.Is(err, ErrBucketRetained) { |
| // Reported, not hidden: the bucket was left to another |
| // account and may hold this store's objects. A person decides. |
| @@ -314,6 +335,22 @@ func simulateBucketName(t *testing.T, seed int64, steps int) (trace []string, pr |
| _ = teardown(a) |
| } |
| ip.between = func(string) {} |
| + if os.Getenv("CARLOS_SIM_SEED") != "" { |
| + // Reproducing one seed: the state after every step. |
| + var states, objects []string |
| + for _, o := range apps { |
| + _, user := mem.Users[UserName(o.acct, o.app)] |
| + states = append(states, fmt.Sprintf("%s=%s user=%v", o.name(), o.state, user)) |
| + } |
| + for key := range mem.Buckets[ledgerBucket] { |
| + objects = append(objects, key) |
| + } |
| + sort.Strings(objects) |
| + ledger, _, _ := ReadBucketOwnership(ctx, s, ledgerBucket) |
| + raw, _ := json.Marshal(ledger) |
| + _, exists := mem.Buckets[ledgerBucket] |
| + logf(" state: %s | bucket exists=%v objects=%v | ledger %s", strings.Join(states, ", "), exists, objects, raw) |
| + } |
| if midProblem != "" { |
| return trace, midProblem |
| } |