amadan
console: the scan enqueue door, behind the consent chain
Task 11, spec §5.1. `POST .../store/scan/object` takes a key and an optional callback path, writes one queue item, and answers 202 without scanning anything. Scanning inline would hold a console goroutine for as long as an engine takes on an object of the tenant's choosing, which hands one app a way to occupy all of them. THE CONSENT CHAIN RUNS HERE, and this is the door where leaving it out was the hole review 3 found. The sweep will not read a tenant's bytes without the member's attestation, and `scan grant` will not start on an operator's say-so alone — so a door checking only the deployment switch would let an app's own credential make the console fetch and scan objects in a store that was never requested, never granted, or attested client-encrypted. It calls objstore's QueuePermitted rather than re-deriving the six conditions, which is why that is now exported: two callers deriving the same fact from the same data is the shape in which they start disagreeing, and §5.1.1 is precisely that they must not. `Service`, not `Admin`. Every other scan verb decides POLICY — who may scan, whether enforcement is on — and belongs to a person. This one asks for work inside a policy somebody already set, and can add nothing an operator has not already granted. The refusal mapping is a TABLE rather than a switch inside the handler, and that is a testability decision, not a tidiness one. Two of its arms cannot be reached through the door: a trashed app is a 404 at apiAccountApp before the chain is consulted, and a refusal name objstore has not defined yet does not exist to send. Both were mutation-tested through HTTP and both SURVIVED — not because the code was wrong, but because no request can distinguish them. Moved out, they are asserted where they are observable, including the one that matters: an unrecognised refusal must refuse. Costing a working feature is survivable; letting an object through on a name the door did not recognise is not. Strict decoding, for one reason: an app that misspells `callback_path` would otherwise be answered 202 for an item carrying no callback, and would wait forever for a delivery nothing ever promised. A door whose optional field is a PROMISE must not accept a body it half-understands. The engine check is last. An app learns why ITS store cannot be scanned before it learns anything about the deployment it happens to run on — and with no engine wired the 202 would be a lie, because nothing would ever drain the item. Nine mutations, all caught. One of them only after a false result: the first run patched an identical `allowWrite` line in handleAPIScanRequest earlier in the same file and reported SURVIVED for a limiter that was never touched. Anchors are checked for uniqueness now, not just for having changed something.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ExnZtUYmt2h8QFcymjPaSL
6 files changed,
+654
−8
internal/console/apiscan.go+187 −0internal/console/apiscanqueue_test.go+429 −0internal/console/authcli.go+9 −0internal/console/server.go+15 −0internal/objstore/queue.go+10 −4internal/objstore/queue_test.go+4 −4
diff --git a/internal/console/apiscan.go b/internal/console/apiscan.go| index c08d6f06d..1fd16dc21 100644 |
| --- a/internal/console/apiscan.go |
| +++ b/internal/console/apiscan.go |
| @@ -26,6 +26,7 @@ import ( |
| "errors" |
| "log" |
| "net/http" |
| + "strings" |
| "time" |
| "github.com/carlosframework/platform/internal/objstore" |
| @@ -621,3 +622,189 @@ func (s *Server) replyScan(w http.ResponseWriter, r *http.Request, acct *Account |
| } |
| writeJSON(w, http.StatusOK, reply) |
| } |
| + |
| +// ---- the on-demand enqueue door (spec §5.1) --------------------------------- |
| + |
| +// scanEnqueueBody is the request. Two fields, and the second is optional: an |
| +// app with no platform-assigned host, or one that would rather poll, sends |
| +// only the key. |
| +type scanEnqueueBody struct { |
| + Key string `json:"key"` |
| + CallbackPath string `json:"callback_path"` |
| +} |
| + |
| +// scanUnavailable writes §5.1's refusal shape: a machine-readable reason AND |
| +// a sentence. |
| +// |
| +// The reason is what an app branches on — "not_granted" and "disabled" want |
| +// different behaviour from a client, and neither is a retry. The sentence is |
| +// for the person reading the log when the client did the wrong thing anyway, |
| +// and it uses the same "error" key every other door on this console uses, so |
| +// a generic client that only knows how to print an error still prints |
| +// something true. |
| +func scanUnavailable(w http.ResponseWriter, status int, reason, msg string) { |
| + writeJSON(w, status, map[string]string{ |
| + "state": "unavailable", |
| + "reason": reason, |
| + "error": msg, |
| + }) |
| +} |
| + |
| +// scanAnswer is how one QueueRefusal is answered at the enqueue door: a |
| +// status and a sentence, or the family's 404. |
| +type scanAnswer struct { |
| + status int |
| + notFound bool |
| + unknown bool |
| + msg string |
| +} |
| + |
| +func (a scanAnswer) refused() bool { return a.status != 0 || a.notFound } |
| + |
| +// scanRefusalAnswer maps every name the consent chain can produce onto this |
| +// door's answer. |
| +// |
| +// A TABLE, not a switch inside the handler, and the reason is testability |
| +// rather than tidiness: the two arms that matter most are the two the handler |
| +// cannot be driven into. A trashed app is already a 404 at apiAccountApp, so |
| +// the trashed arm is unreachable through HTTP; an unrecognised name does not |
| +// exist yet, by definition. Left inline, both would be branches nothing could |
| +// assert — and an unasserted refusal branch is exactly where "the door that |
| +// did not recognise it let it through" gets written. |
| +// |
| +// The default is a REFUSAL, so a name added to objstore and not taught to |
| +// this door costs a working feature rather than an unscanned object. |
| +func scanRefusalAnswer(r objstore.QueueRefusal) scanAnswer { |
| + switch r { |
| + case objstore.RefusalNone: |
| + return scanAnswer{} |
| + case objstore.RefusalTrashed: |
| + // The SAME 404 apiAccountApp already gives, so that if the two |
| + // checks ever drift apart this door does not become the one that |
| + // confirms an app exists by refusing differently. |
| + return scanAnswer{notFound: true} |
| + case objstore.RefusalNotGranted: |
| + return scanAnswer{status: http.StatusConflict, |
| + msg: "this store has not been granted yet — an operator runs `carlos store grant`"} |
| + case objstore.RefusalNotRequested: |
| + return scanAnswer{status: http.StatusConflict, |
| + msg: "scanning has not been requested for this store — a member runs `carlos store scan request`"} |
| + case objstore.RefusalEncrypted: |
| + return scanAnswer{status: http.StatusConflict, |
| + msg: "this store is attested client-encrypted, so a verdict on its bytes would mean nothing"} |
| + case objstore.RefusalNotEnabled: |
| + return scanAnswer{status: http.StatusConflict, |
| + msg: "scanning has not been granted for this store — an operator runs `carlos store scan grant`"} |
| + case objstore.RefusalDisabled: |
| + return scanAnswer{status: http.StatusServiceUnavailable, |
| + msg: "object scanning is disabled for this deployment — an operator runs `carlos system antivirus enable`"} |
| + default: |
| + return scanAnswer{status: http.StatusConflict, unknown: true, |
| + msg: "this store may not be scanned right now — run `carlos store scan status`"} |
| + } |
| +} |
| + |
| +// handleAPIScanEnqueue asks for one object to be scanned soon, and answers |
| +// 202 without scanning it. |
| +// |
| +// IT ENQUEUES; IT DOES NOT SCAN. Scanning inline would hold a console |
| +// goroutine for as long as an engine takes on an object of the tenant's |
| +// choosing, which hands one app a trivial way to occupy all of them. The |
| +// drain reads the item on its own clock (§7.2). |
| +// |
| +// THE CONSENT CHAIN RUNS HERE TOO, and this is the door where forgetting it |
| +// was the hole review 3 found. The sweep will not read a tenant's bytes |
| +// without the member's attestation, and `scan grant` will not start on an |
| +// operator's say-so alone — so a door that checked only the deployment switch |
| +// would let an app's own credential make the console fetch and scan objects |
| +// in a store that was never requested, never granted, or attested as |
| +// client-encrypted. Every one of those refusals is objstore's |
| +// QueuePermitted, not a re-derivation here. |
| +// |
| +// ORDER: the store's own state before the deployment's. An app learns why ITS |
| +// store cannot be scanned before it learns anything about the deployment it |
| +// happens to be running on, and the two 503s below stay the last word. |
| +func (s *Server) handleAPIScanEnqueue(w http.ResponseWriter, r *http.Request) { |
| + addr, acct, app, ok := s.apiAccountApp(w, r) |
| + if !ok { |
| + return |
| + } |
| + p, _ := principalFrom(r.Context()) |
| + if !s.allowWrite(p, time.Now()) { |
| + jsonError(w, http.StatusTooManyRequests, "too many requests — wait a minute and try again") |
| + return |
| + } |
| + // STRICT, and this is the door that needs it: an app that misspells |
| + // callback_path would otherwise be answered 202 for an item carrying no |
| + // callback at all, and would then wait for a delivery nothing ever |
| + // promised. A door whose optional field is a PROMISE must not accept a |
| + // body it half-understands. |
| + var body scanEnqueueBody |
| + if !readJSONStrictLimit(w, r, &body, 8<<10) { |
| + return |
| + } |
| + key := strings.TrimSpace(body.Key) |
| + if key == "" { |
| + jsonError(w, http.StatusBadRequest, |
| + `name the object to scan, e.g. {"key": "uploads/x.pdf"}`) |
| + return |
| + } |
| + if body.CallbackPath != "" { |
| + if err := validCallbackPath(body.CallbackPath); err != nil { |
| + // The validator's own words. They name the shape that was |
| + // refused ("protocol-relative", "'..' segment"), which is what |
| + // the app author needs, and none of them echoes the input back |
| + // into the response. |
| + jsonError(w, http.StatusBadRequest, err.Error()+ |
| + " — callback_path is a path on your app's platform host, not a URL") |
| + return |
| + } |
| + } |
| + if s.cfg.Releases == nil { |
| + scanNoEngine(w) |
| + return |
| + } |
| + ctx, cancel := context.WithTimeout(r.Context(), storeTimeout) |
| + defer cancel() |
| + |
| + // The record itself is not wanted here. The door's whole use of the |
| + // consent chain is the NAME it comes back with — QueuePermitted returns |
| + // a record only for the one reversible refusal, and this door has |
| + // nothing to say about a store it is refusing. |
| + _, refusal, err := objstore.QueuePermitted(ctx, s.cfg.Releases, acct.Sqid, app) |
| + if err != nil { |
| + log.Printf("console: api scan enqueue %s/%s: consent: %v", acct.Sqid, app, err) |
| + jsonError(w, http.StatusBadGateway, "the store record could not be read — try again") |
| + return |
| + } |
| + if ans := scanRefusalAnswer(refusal); ans.refused() { |
| + if ans.unknown { |
| + log.Printf("console: api scan enqueue %s/%s: unhandled refusal %q", acct.Sqid, app, refusal) |
| + } |
| + if ans.notFound { |
| + http.NotFound(w, r) |
| + return |
| + } |
| + scanUnavailable(w, ans.status, string(refusal), ans.msg) |
| + return |
| + } |
| + // LAST, because it is the deployment's state rather than this store's, |
| + // and because it is the one refusal that is about whether the queue can |
| + // ever drain: with no engine wired the item would sit until its TTL and |
| + // the 202 would have been a lie. |
| + if s.cfg.StoreScanner == nil { |
| + scanNoEngine(w) |
| + return |
| + } |
| + if _, err := objstore.Enqueue(ctx, s.cfg.Releases, s.scanQueue, acct.Sqid, app, |
| + &objstore.QueueItem{Key: key, By: addr, CallbackPath: body.CallbackPath}, |
| + time.Now()); err != nil { |
| + log.Printf("console: api scan enqueue %s/%s %q: %v", acct.Sqid, app, key, err) |
| + jsonError(w, http.StatusBadGateway, "the scan could not be queued — try again") |
| + return |
| + } |
| + // NO ID IN THE REPLY. The status door (§5.2) is keyed by object key, not |
| + // by queue id, so an id here would be a handle with nothing to hand it |
| + // to — and one clients would build on before it meant anything. |
| + writeJSON(w, http.StatusAccepted, map[string]string{"state": "pending"}) |
| +} |
diff --git a/internal/console/apiscanqueue_test.go b/internal/console/apiscanqueue_test.go| new file mode 100644 |
| index 000000000..4001640f6 |
| --- /dev/null |
| +++ b/internal/console/apiscanqueue_test.go |
| @@ -0,0 +1,429 @@ |
| +package console |
| + |
| +import ( |
| + "context" |
| + "net/http" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/objstore" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/svcrole" |
| +) |
| + |
| +func enqueueURL(fx *consoleFixture, app string) string { |
| + return scanURL(fx, app) + "/object" |
| +} |
| + |
| +// appToken is the credential this door is FOR: an app's own service principal, |
| +// bound to one app, holding the Service role and nothing else. |
| +func appToken(t *testing.T, fx *consoleFixture, name string) string { |
| + t.Helper() |
| + return serviceToken(t, fx.st, OpsAccountID, name, "titogo", svcrole.Service) |
| +} |
| + |
| +// queuedKeys is what actually landed in the bucket. The door's 202 is a |
| +// promise about an object, so every test that reads a 202 checks the object, |
| +// and every test that reads a refusal checks that there is none — a refusal |
| +// that still enqueued would be the worst of both. |
| +func queuedKeys(t *testing.T, fx *consoleFixture, app string) []string { |
| + t.Helper() |
| + keys, err := fx.rel.List(context.Background(), objstore.QueuePrefix(release.OpsAccount, app)) |
| + if err != nil { |
| + t.Fatalf("list queue: %v", err) |
| + } |
| + return keys |
| +} |
| + |
| +// enqueueArmed is the whole consent chain satisfied and an engine wired: the |
| +// only state in which this door says 202. |
| +func enqueueArmed(t *testing.T, code string) (*consoleFixture, string) { |
| + t.Helper() |
| + fx, _, token, _ := storeArmed(t, code) |
| + grantStore(t, fx, token) |
| + requestScanning(t, fx, token, false) |
| + // Engine, then the deployment switch, THEN the per-store grant. That is |
| + // the real sequence and the order is not incidental: `scan grant` refuses |
| + // with no engine wired (there is nothing to enable) and refuses again |
| + // while the deployment switch is off (it would be enabling a control that |
| + // is not running). |
| + fx.srv.cfg.StoreScanner = objstore.NewMemScanner() |
| + enableDeploymentScanning(t, fx) |
| + fx.srv.cfg.StoreOperators = []string{"alice@keymail.dev"} |
| + if s, o := postJSON(t, scanURL(fx, "titogo")+"/grant", map[string]any{}, token); s != http.StatusOK { |
| + t.Fatalf("scan grant = %d %v", s, o) |
| + } |
| + return fx, appToken(t, fx, "titobot") |
| +} |
| + |
| +// TestEnqueueAcceptsAndWritesTheItem is the positive control every refusal |
| +// test below is measured against. Without it a door that refused EVERYTHING |
| +// would pass the entire rest of this file. |
| +func TestEnqueueAcceptsAndWritesTheItem(t *testing.T) { |
| + fx, tok := enqueueArmed(t, "code-enq-ok") |
| + |
| + status, out := postJSON(t, enqueueURL(fx, "titogo"), |
| + map[string]any{"key": "uploads/x.pdf", "callback_path": "/hooks/scan"}, tok) |
| + if status != http.StatusAccepted { |
| + t.Fatalf("enqueue = %d %v, want 202", status, out) |
| + } |
| + if out["state"] != "pending" { |
| + t.Errorf("state = %v, want pending", out["state"]) |
| + } |
| + // No id in the reply: the status door is keyed by object key, so an id |
| + // here would be a handle with nothing to hand it to. |
| + if _, ok := out["id"]; ok { |
| + t.Errorf("reply carries an id (%v) — nothing consumes one, and a client would build on it", out) |
| + } |
| + keys := queuedKeys(t, fx, "titogo") |
| + if len(keys) != 1 { |
| + t.Fatalf("queued %d items, want 1: %v", len(keys), keys) |
| + } |
| + // And the index knows, or the drain never looks at this app. |
| + if n := fx.srv.scanQueue.Len(); n != 1 { |
| + t.Errorf("index holds %d apps, want 1 — an item nothing points at is stranded until the next boot", n) |
| + } |
| +} |
| + |
| +// TestEnqueueRefusesEveryConsentState is §5.1.1's chain, one case per name. |
| +// |
| +// The states are built by STOPPING the real sequence at each step rather than |
| +// by writing records by hand: a fixture that hand-rolls "granted but not |
| +// requested" can drift from what the grant door actually writes, and then the |
| +// test passes against a shape the deployment never produces. |
| +func TestEnqueueRefusesEveryConsentState(t *testing.T) { |
| + for _, tc := range []struct { |
| + name string |
| + reason string |
| + status int |
| + // arm builds the state and returns the app's token. |
| + arm func(t *testing.T, fx *consoleFixture, member string) string |
| + }{{ |
| + name: "no store at all", |
| + reason: "not_granted", |
| + status: http.StatusConflict, |
| + arm: func(t *testing.T, fx *consoleFixture, member string) string { |
| + return appToken(t, fx, "bot") |
| + }, |
| + }, { |
| + name: "store granted, scanning never requested", |
| + reason: "not_requested", |
| + status: http.StatusConflict, |
| + arm: func(t *testing.T, fx *consoleFixture, member string) string { |
| + grantStore(t, fx, member) |
| + return appToken(t, fx, "bot") |
| + }, |
| + }, { |
| + name: "requested, but attested client-encrypted", |
| + reason: "encrypted", |
| + status: http.StatusConflict, |
| + arm: func(t *testing.T, fx *consoleFixture, member string) string { |
| + grantStore(t, fx, member) |
| + requestScanning(t, fx, member, true) |
| + return appToken(t, fx, "bot") |
| + }, |
| + }, { |
| + name: "requested, but no operator grant", |
| + reason: "not_enabled", |
| + status: http.StatusConflict, |
| + arm: func(t *testing.T, fx *consoleFixture, member string) string { |
| + grantStore(t, fx, member) |
| + requestScanning(t, fx, member, false) |
| + return appToken(t, fx, "bot") |
| + }, |
| + }, { |
| + name: "everything granted, deployment switched off", |
| + reason: "disabled", |
| + status: http.StatusServiceUnavailable, |
| + arm: func(t *testing.T, fx *consoleFixture, member string) string { |
| + grantStore(t, fx, member) |
| + requestScanning(t, fx, member, false) |
| + fx.srv.cfg.StoreScanner = objstore.NewMemScanner() |
| + enableDeploymentScanning(t, fx) |
| + fx.srv.cfg.StoreOperators = []string{"alice@keymail.dev"} |
| + if s, o := postJSON(t, scanURL(fx, "titogo")+"/grant", map[string]any{}, member); s != http.StatusOK { |
| + t.Fatalf("scan grant = %d %v", s, o) |
| + } |
| + // Granted, and THEN switched off: the one refusal that is |
| + // reversible, and the only one where the store itself is ready. |
| + disableDeploymentScanning(t, fx) |
| + return appToken(t, fx, "bot") |
| + }, |
| + }} { |
| + t.Run(tc.name, func(t *testing.T) { |
| + fx, _, member, _ := storeArmed(t, "code-enq-"+strings.ReplaceAll(tc.name, " ", "-")) |
| + tok := tc.arm(t, fx, member) |
| + |
| + status, out := postJSON(t, enqueueURL(fx, "titogo"), |
| + map[string]any{"key": "uploads/x.pdf"}, tok) |
| + if status != tc.status { |
| + t.Fatalf("enqueue = %d %v, want %d", status, out, tc.status) |
| + } |
| + if out["state"] != "unavailable" { |
| + t.Errorf("state = %v, want unavailable", out["state"]) |
| + } |
| + if out["reason"] != tc.reason { |
| + t.Errorf("reason = %v, want %q", out["reason"], tc.reason) |
| + } |
| + // A reason with no sentence beside it leaves whoever reads the |
| + // log with a token and no next step. |
| + if msg, _ := out["error"].(string); strings.TrimSpace(msg) == "" { |
| + t.Errorf("no error sentence beside reason %q: %v", tc.reason, out) |
| + } |
| + if keys := queuedKeys(t, fx, "titogo"); len(keys) != 0 { |
| + t.Errorf("refused with %q and still queued %v", tc.reason, keys) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// TestEnqueueRefusesWithoutAttestation is the hole review 3 found, on its own |
| +// rather than as one row of the table above, because it is the one a future |
| +// change is most likely to reintroduce. |
| +// |
| +// Everything the DEPLOYMENT controls is in place: an engine is wired, the |
| +// switch is on, the store is granted and provisioned. The only thing missing |
| +// is the member's own `carlos store scan request` — the attestation that says |
| +// whether these bytes are readable at all. A door that checked the deployment |
| +// switch and stopped would go straight through, and the console would fetch |
| +// and scan a tenant's objects on the say-so of that tenant's own app. |
| +func TestEnqueueRefusesWithoutAttestation(t *testing.T) { |
| + fx, _, member, _ := storeArmed(t, "code-enq-attest") |
| + grantStore(t, fx, member) |
| + fx.srv.cfg.StoreScanner = objstore.NewMemScanner() |
| + enableDeploymentScanning(t, fx) |
| + tok := appToken(t, fx, "titobot") |
| + |
| + status, out := postJSON(t, enqueueURL(fx, "titogo"), |
| + map[string]any{"key": "uploads/x.pdf"}, tok) |
| + if status != http.StatusConflict || out["reason"] != "not_requested" { |
| + t.Fatalf("enqueue = %d %v, want 409 not_requested — the sweep will not read these bytes either", status, out) |
| + } |
| + if keys := queuedKeys(t, fx, "titogo"); len(keys) != 0 { |
| + t.Fatalf("queued %v without the member's attestation", keys) |
| + } |
| +} |
| + |
| +// TestEnqueueRequiresServiceRole pins the role, both ways. |
| +// |
| +// Admin reaching it is stated rather than fought: svcrole.Admin is the union |
| +// of every role by construction, so a test asserting Admin is refused here |
| +// would be asserting the opposite of how the role set works, and the next |
| +// person would "fix" the role table to satisfy it. |
| +func TestEnqueueRequiresServiceRole(t *testing.T) { |
| + fx, _ := enqueueArmed(t, "code-enq-role") |
| + |
| + for _, tc := range []struct { |
| + role svcrole.Role |
| + want int |
| + }{ |
| + {svcrole.Publish, http.StatusForbidden}, |
| + {svcrole.Instance, http.StatusForbidden}, |
| + {svcrole.Operate, http.StatusForbidden}, |
| + {svcrole.Service, http.StatusAccepted}, |
| + {svcrole.Admin, http.StatusAccepted}, |
| + } { |
| + t.Run(string(tc.role), func(t *testing.T) { |
| + other := serviceToken(t, fx.st, OpsAccountID, "bot-"+string(tc.role), "titogo", tc.role) |
| + status, out := postJSON(t, enqueueURL(fx, "titogo"), |
| + map[string]any{"key": "uploads/" + string(tc.role) + ".pdf"}, other) |
| + if status != tc.want { |
| + t.Fatalf("role %s = %d %v, want %d", tc.role, status, out, tc.want) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// TestEnqueueRefusesAHostileCallbackPath proves the DOOR calls the validator. |
| +// |
| +// scanhook_test.go proves the validator is right about each shape; that is a |
| +// different claim from "nothing hostile can be stored", which is only true if |
| +// the door runs it on the way in. One row per family is enough here — the |
| +// exhaustive table belongs where the rule lives — and what this asserts on |
| +// top of the 400 is that the bucket stayed empty. |
| +func TestEnqueueRefusesAHostileCallbackPath(t *testing.T) { |
| + fx, tok := enqueueArmed(t, "code-enq-path") |
| + |
| + for _, p := range []string{ |
| + "https://evil.example/x", // absolute |
| + "//evil.example/x", // protocol-relative |
| + "/\\evil.example/x", // backslash, read as protocol-relative by enough parsers |
| + "/hooks/../../admin", // traversal |
| + "/hooks\r\nX-Evil: 1", // header injection |
| + "/hooks%0d%0aX-Evil:1", // and its encoded form |
| + "/hooks#frag", // fragment |
| + "/x@evil.example/y", // smuggled authority |
| + "/" + strings.Repeat("a", maxCallbackPath), // over the cap |
| + } { |
| + t.Run(p, func(t *testing.T) { |
| + status, out := postJSON(t, enqueueURL(fx, "titogo"), |
| + map[string]any{"key": "uploads/x.pdf", "callback_path": p}, tok) |
| + if status != http.StatusBadRequest { |
| + t.Fatalf("callback_path %q = %d %v, want 400", p, status, out) |
| + } |
| + if keys := queuedKeys(t, fx, "titogo"); len(keys) != 0 { |
| + t.Fatalf("stored an item carrying %q: %v", p, keys) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// TestEnqueueRefusesABodyItHalfUnderstands is why this door decodes strictly. |
| +// |
| +// An app that misspells callback_path would otherwise be answered 202 for an |
| +// item carrying no callback, and would wait forever for a delivery nobody ever |
| +// promised. The 400 names the field, which is the whole of the fix. |
| +func TestEnqueueRefusesABodyItHalfUnderstands(t *testing.T) { |
| + fx, tok := enqueueArmed(t, "code-enq-strict") |
| + |
| + status, out := postJSON(t, enqueueURL(fx, "titogo"), |
| + map[string]any{"key": "uploads/x.pdf", "callback_url": "/hooks/scan"}, tok) |
| + if status != http.StatusBadRequest { |
| + t.Fatalf("unknown field = %d %v, want 400", status, out) |
| + } |
| + if msg, _ := out["error"].(string); !strings.Contains(msg, "callback_url") { |
| + t.Errorf("error %q does not name the field the caller got wrong", msg) |
| + } |
| + if keys := queuedKeys(t, fx, "titogo"); len(keys) != 0 { |
| + t.Fatalf("queued %v from a body it did not fully understand", keys) |
| + } |
| +} |
| + |
| +// TestEnqueueNeedsAKey: a 202 is a promise about an object, and there is no |
| +// object here. |
| +func TestEnqueueNeedsAKey(t *testing.T) { |
| + fx, tok := enqueueArmed(t, "code-enq-nokey") |
| + |
| + for _, key := range []string{"", " "} { |
| + status, out := postJSON(t, enqueueURL(fx, "titogo"), map[string]any{"key": key}, tok) |
| + if status != http.StatusBadRequest { |
| + t.Fatalf("key %q = %d %v, want 400", key, status, out) |
| + } |
| + } |
| + if keys := queuedKeys(t, fx, "titogo"); len(keys) != 0 { |
| + t.Fatalf("queued %v with no key", keys) |
| + } |
| +} |
| + |
| +// TestEnqueueRefusesWithNoEngineWired is the 503 that keeps the 202 honest: |
| +// with no engine the drain can never run, so an accepted item would sit until |
| +// its TTL and the app would wait on an answer that was never coming. |
| +// |
| +// It is checked LAST, after the consent chain, so an app learns why ITS store |
| +// cannot be scanned before it learns anything about the deployment. |
| +func TestEnqueueRefusesWithNoEngineWired(t *testing.T) { |
| + fx, tok := enqueueArmed(t, "code-enq-noengine") |
| + fx.srv.cfg.StoreScanner = nil |
| + |
| + status, out := postJSON(t, enqueueURL(fx, "titogo"), |
| + map[string]any{"key": "uploads/x.pdf"}, tok) |
| + if status != http.StatusServiceUnavailable { |
| + t.Fatalf("enqueue with no engine = %d %v, want 503", status, out) |
| + } |
| + if keys := queuedKeys(t, fx, "titogo"); len(keys) != 0 { |
| + t.Fatalf("queued %v with no engine to drain it", keys) |
| + } |
| +} |
| + |
| +// TestEnqueueIsRateLimited: the door writes an object per call, so an app in a |
| +// loop is an app filling its own bucket. 429 is the same limiter every other |
| +// write door on this console shares. |
| +func TestEnqueueIsRateLimited(t *testing.T) { |
| + fx, tok := enqueueArmed(t, "code-enq-429") |
| + |
| + saw429 := false |
| + for i := 0; i < 200; i++ { |
| + status, _ := postRaw(t, enqueueURL(fx, "titogo"), |
| + map[string]any{"key": "uploads/x.pdf"}, tok) |
| + if status == http.StatusTooManyRequests { |
| + saw429 = true |
| + break |
| + } |
| + if status != http.StatusAccepted { |
| + t.Fatalf("enqueue #%d = %d, want 202 until the limiter bites", i, status) |
| + } |
| + } |
| + if !saw429 { |
| + t.Fatal("200 enqueues and no 429 — this door writes an object per call") |
| + } |
| +} |
| + |
| +// TestEnqueueIsScopedToItsOwnApp: an app-bound service credential reaches |
| +// exactly one app, and the refusal is the family's 404 rather than a 403 that |
| +// would confirm the other app exists. |
| +func TestEnqueueIsScopedToItsOwnApp(t *testing.T) { |
| + fx, _ := enqueueArmed(t, "code-enq-scope") |
| + if _, err := fx.st.CreateAppIn(OpsAccountID, "otherapp", "paul@keymail.dev", time.Now()); err != nil { |
| + t.Fatal(err) |
| + } |
| + tok := appToken(t, fx, "scopedbot") |
| + |
| + status, _ := postRaw(t, enqueueURL(fx, "otherapp"), map[string]any{"key": "uploads/x.pdf"}, tok) |
| + if status != http.StatusNotFound { |
| + t.Fatalf("cross-app enqueue = %d, want 404", status) |
| + } |
| +} |
| + |
| +// TestScanRefusalAnswerCoversEveryName is the table's own test, and it exists |
| +// because two of its arms cannot be reached through the door. |
| +// |
| +// A trashed app is a 404 at apiAccountApp before the consent chain is |
| +// consulted, and a refusal name objstore has not defined yet does not exist to |
| +// send. Both were mutation-tested through HTTP and both survived — not because |
| +// the code was wrong, but because no request can distinguish them. So they are |
| +// asserted where they ARE observable. |
| +// |
| +// The list is written out rather than derived: Go cannot enumerate the |
| +// constants of a string type, so a derived list would be this list anyway, |
| +// with the compiler no longer checking that each name still exists. |
| +func TestScanRefusalAnswerCoversEveryName(t *testing.T) { |
| + named := []objstore.QueueRefusal{ |
| + objstore.RefusalNotGranted, |
| + objstore.RefusalTrashed, |
| + objstore.RefusalNotRequested, |
| + objstore.RefusalEncrypted, |
| + objstore.RefusalNotEnabled, |
| + objstore.RefusalDisabled, |
| + } |
| + for _, r := range named { |
| + ans := scanRefusalAnswer(r) |
| + if !ans.refused() { |
| + t.Errorf("%q: the door does not refuse it — a consent refusal that enqueues is the hole this chain exists to close", r) |
| + } |
| + if ans.unknown { |
| + t.Errorf("%q: answered as an unrecognised name, so the door has not been taught it", r) |
| + } |
| + if ans.notFound { |
| + continue // the 404 arm carries no sentence, by design |
| + } |
| + if ans.status < 400 { |
| + t.Errorf("%q: status %d is not a refusal", r, ans.status) |
| + } |
| + if strings.TrimSpace(ans.msg) == "" { |
| + t.Errorf("%q: refused with no sentence beside the reason", r) |
| + } |
| + } |
| + |
| + // The trashed arm answers as the family's 404 and says nothing else. A |
| + // reason here would make this door the one that confirms an app exists. |
| + if ans := scanRefusalAnswer(objstore.RefusalTrashed); !ans.notFound || ans.status != 0 { |
| + t.Errorf("trashed = %+v, want the bare 404 apiAccountApp already gives", ans) |
| + } |
| + |
| + // RefusalNone is the ONLY value that means yes. |
| + if ans := scanRefusalAnswer(objstore.RefusalNone); ans.refused() { |
| + t.Errorf("RefusalNone = %+v, want no refusal at all", ans) |
| + } |
| + |
| + // A name objstore might add tomorrow. The door has never seen it, and the |
| + // answer must still be a refusal: costing a working feature is survivable, |
| + // letting an unscanned object through on an unrecognised refusal is not. |
| + ans := scanRefusalAnswer(objstore.QueueRefusal("some_future_reason")) |
| + if !ans.refused() { |
| + t.Fatal("an unrecognised refusal enqueues — a name added to objstore would silently open this door") |
| + } |
| + if !ans.unknown { |
| + t.Error("an unrecognised refusal is not flagged unknown, so nothing would be logged about it") |
| + } |
| +} |
diff --git a/internal/console/authcli.go b/internal/console/authcli.go| index c46294bd6..c6e05b4b5 100644 |
| --- a/internal/console/authcli.go |
| +++ b/internal/console/authcli.go |
| @@ -756,6 +756,15 @@ func (s *Server) registerCLIRoutes() { |
| s.cliRoute("POST /api/cli/apps/{account}/{app}/store/scan/revoke", svcrole.Allow(svcrole.Admin), s.handleAPIScanRevoke) |
| s.cliRoute("POST /api/cli/apps/{account}/{app}/store/scan/sweep", svcrole.Allow(svcrole.Admin), s.handleAPIScanSweep) |
| s.cliRoute("POST /api/cli/apps/{account}/{app}/store/scan/enforce", svcrole.Allow(svcrole.Admin), s.handleAPIScanEnforce) |
| + // The ONE scan door an app's own credential reaches: "scan this object |
| + // now" (spec §5.1). Service, not Admin, and the difference is the point — |
| + // every verb above decides POLICY (who may scan, whether enforcement is |
| + // on) and belongs to a person, while this one asks for work inside a |
| + // policy somebody already set. It can add nothing an operator has not |
| + // already granted: the consent chain in the handler refuses unless the |
| + // store was requested, granted and enabled, so the worst a stolen service |
| + // token achieves is queueing scans of its own app's objects. |
| + s.cliRoute("POST /api/cli/apps/{account}/{app}/store/scan/object", svcrole.Allow(svcrole.Service), s.handleAPIScanEnqueue) |
| // The deployment-wide doors (internal/console/apisystem.go). No {account} |
| // segment, so no service credential can be scoped to them at all: an empty |
| // Set shuts them to every credential, and closedDoors pins that shut. |
diff --git a/internal/console/server.go b/internal/console/server.go| index 52a89ce80..f1e393f74 100644 |
| --- a/internal/console/server.go |
| +++ b/internal/console/server.go |
| @@ -681,6 +681,20 @@ type Server struct { |
| // sweep is driven against a MemProvisioner's buckets with no S3 and no |
| // credentials. |
| storeListerFor func(ctx context.Context, p *objstore.Provisioned) (objstore.ObjectLister, error) |
| + |
| + // scanQueue is which apps have on-demand scan work waiting (spec §7.1). |
| + // |
| + // SOFT STATE, and never the queue itself: the items are objects in each |
| + // app's own prefix, and this only says where to look. Losing it costs |
| + // latency, not items — the boot rebuild finds anything stranded and the |
| + // hourly sweep reaches every object regardless. |
| + // |
| + // Built in NewServer rather than lazily, because both writers reach it |
| + // without coordination: the enqueue door bumps it from a request |
| + // goroutine while the drain reads it from its own. A nil map assigned on |
| + // first use would be the race, and a Server that always has one cannot |
| + // have it. |
| + scanQueue *objstore.QueueIndex |
| // ceCosts is billPass's test/fake seam for Cost Explorer (cepull.go, |
| // economicspass.go): nil means "use the real, lazily-built CE client" |
| // (s.realCECosts); tests inject a fake so a pull can be proven to |
| @@ -961,6 +975,7 @@ func NewServer(cfg Config, st *Store) (*Server, error) { |
| delegationZone: normalizeDelegationZone(cfg.DelegationZone), |
| steeringHost: normalizeSteeringHost(cfg.SteeringHost), |
| lookupCNAME: net.DefaultResolver.LookupCNAME, |
| + scanQueue: objstore.NewQueueIndex(), |
| } |
| if s.km != nil { |
| s.sendApp = s.km.SendApp |
diff --git a/internal/objstore/queue.go b/internal/objstore/queue.go| index 17674eb8d..f1bdda53e 100644 |
| --- a/internal/objstore/queue.go |
| +++ b/internal/objstore/queue.go |
| @@ -273,7 +273,7 @@ func DrainApp(ctx context.Context, s release.Store, sc Scanner, idx *QueueIndex, |
| // The consent chain, re-checked here and not only at the door: the record |
| // can change between enqueue and drain, and THIS is where bytes are read. |
| - rec, refusal, err := queuePermitted(ctx, s, acct, app) |
| + rec, refusal, err := QueuePermitted(ctx, s, acct, app) |
| if err != nil { |
| return nil, false, err |
| } |
| @@ -477,7 +477,7 @@ func drainOne(ctx context.Context, s release.Store, sc Scanner, acct, app, bucke |
| // "will this item ever be useful" but "may this app be scanned right now", |
| // and a result object written after the answer turned to no is a verdict |
| // no door may answer from. |
| - if _, refusal, err := queuePermitted(ctx, s, acct, app); err != nil || refusal != RefusalNone { |
| + if _, refusal, err := QueuePermitted(ctx, s, acct, app); err != nil || refusal != RefusalNone { |
| return out, err |
| } |
| if err := WriteScanResult(ctx, s, acct, app, &ScanResult{ |
| @@ -489,9 +489,15 @@ func drainOne(ctx context.Context, s release.Store, sc Scanner, acct, app, bucke |
| return out, nil |
| } |
| -// queuePermitted is the consent chain, in one place so the door and the drain |
| +// QueuePermitted is the consent chain, in one place so the door and the drain |
| // cannot answer it differently. |
| -func queuePermitted(ctx context.Context, s release.Store, acct, app string) (*Record, QueueRefusal, error) { |
| +// |
| +// EXPORTED because the door lives in another package. That is the whole |
| +// reason, and it is worth stating: the alternative — console re-deriving the |
| +// same six conditions from a record it fetched itself — is the exact shape |
| +// §5.1.1 exists to forbid. One implementation, two callers, and the only |
| +// difference between them is what they do with the answer. |
| +func QueuePermitted(ctx context.Context, s release.Store, acct, app string) (*Record, QueueRefusal, error) { |
| rec, _, err := Get(ctx, s, acct, app) |
| if err != nil { |
| return nil, RefusalNone, err |
diff --git a/internal/objstore/queue_test.go b/internal/objstore/queue_test.go| index dccb34ab3..833ca50cb 100644 |
| --- a/internal/objstore/queue_test.go |
| +++ b/internal/objstore/queue_test.go |
| @@ -658,7 +658,7 @@ func TestDrainAppKeepsItemOnTransientReadError(t *testing.T) { |
| } |
| // The record itself failing to read is reversible too, and it is the SIBLING |
| -// of the check above: queuePermitted honours "absence, not error" for the |
| +// of the check above: QueuePermitted honours "absence, not error" for the |
| // record, and an earlier draft honoured it there and nowhere else. |
| func TestDrainAppKeepsItemsOnRecordReadError(t *testing.T) { |
| ctx := context.Background() |
| @@ -982,9 +982,9 @@ func TestQueueRefusalNamesEachLinkAndItsConsequence(t *testing.T) { |
| s, _, _, _ := queueFixture(t, 1) |
| tc.snap(t, s) |
| - rec, refusal, err := queuePermitted(ctx, s, "bab", "titogo") |
| + rec, refusal, err := QueuePermitted(ctx, s, "bab", "titogo") |
| if err != nil { |
| - t.Fatalf("queuePermitted: %v", err) |
| + t.Fatalf("QueuePermitted: %v", err) |
| } |
| if refusal != tc.want { |
| t.Fatalf("refusal = %q, want %q", refusal, tc.want) |
| @@ -1017,7 +1017,7 @@ func TestQueuePermittedReportsAReadErrorAsAnError(t *testing.T) { |
| s, _, _, _ := queueFixture(t, 1) |
| broken := &queueStore{Store: s, failGetKey: "apps/bab/titogo/store.json"} |
| - rec, refusal, err := queuePermitted(context.Background(), broken, "bab", "titogo") |
| + rec, refusal, err := QueuePermitted(context.Background(), broken, "bab", "titogo") |
| if err == nil { |
| t.Fatal("a failing store.json read came back as a clean answer") |
| } |