amadan

carlos / platform Public

console: CLI API — multipart ship, hashes computed server-side

`carlos ship` now has a console-API door: POST
/api/cli/apps/{account}/{app}/releases takes a streamed
multipart/form-data body (meta part first, then one part per artifact),
computes each artifact's SHA256/size from the bytes that actually
arrived, and hands them to release.Ship — which itself recomputes and
cross-checks, so nothing here is trusted from the caller.

Gate order: apiAccountApp's byte-identical 404, then a dedicated
per-token rate limit (ships are rare and heavy, unlike the address-keyed
write door), then multipart parsing under http.MaxBytesReader via
r.MultipartReader() (never ParseMultipartForm, whose own buffering limit
would sit outside this one). Part names become bucket key segments, so
targetRe/"site.tar.gz" checks are path-traversal defense.
Paul Campbell 8a876dfcb51b460dd2b2174be6ba5c46d5548348 parent 40e1292
3 files changed, +456 −1
  • internal/console/apicli.go +182 −0
  • internal/console/apicli_test.go +256 −0
  • internal/console/authcli.go +18 −1
diff --git a/internal/console/apicli.go b/internal/console/apicli.go
index e6d4f7a39..948e66760 100644
--- a/internal/console/apicli.go
+++ b/internal/console/apicli.go
@@ -10,14 +10,21 @@ package console
import (
"context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
"fmt"
+ "io"
"log"
"net/http"
"net/url"
+ "regexp"
"strings"
"time"
"github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/version"
)
// apiAccountApp resolves and authorizes one CLI API request. The account
@@ -327,3 +334,178 @@ func (s *Server) handleAPIRollback(w http.ResponseWriter, r *http.Request) {
s.audit(acct, app, addr, "rollback", "rolled back "+channel+" to "+target+" (cli)")
writeJSON(w, http.StatusOK, map[string]string{"result": "rolled back"})
}
+
+// ---- ship (multipart upload) ------------------------------------------------
+
+// apiShipMaxBytes bounds one ship request across every part TOGETHER — meta
+// plus all artifacts — not any single part on its own. A var, not a const,
+// so a test can lower it without rebuilding the fixture. 256MiB comfortably
+// covers today's binaries and a static site tarball in one request.
+var apiShipMaxBytes int64 = 256 << 20
+
+// targetRe validates a binary artifact's part NAME before it is used to
+// build a bucket key: release.ArtifactKey joins it straight into the object
+// path, so this check is path-traversal defense, not tidiness — a part
+// named "../../etc/passwd" must never reach release.Ship as a file name.
+var targetRe = regexp.MustCompile(`^[a-z0-9]+-[a-z0-9]+$`)
+
+// shipMaxBytesRefused answers 413 if err is the http.MaxBytesReader wrapped
+// around the request body cutting a read short, and reports whether it did
+// — the caller returns immediately when this is true, and falls through to
+// its own (more specific) 400 otherwise.
+func shipMaxBytesRefused(w http.ResponseWriter, err error) bool {
+ var tooLarge *http.MaxBytesError
+ if errors.As(err, &tooLarge) {
+ jsonError(w, http.StatusRequestEntityTooLarge, "upload too large")
+ return true
+ }
+ return false
+}
+
+// handleAPIShip answers `carlos ship`: a multipart upload of one release's
+// artifacts. Hashes and sizes are computed HERE, from the bytes that
+// actually arrived on the wire, and handed to release.Ship as
+// already-filled Artifact{SHA256,Size} — Ship itself recomputes and
+// cross-checks them (release.go), so there is no way for a caller to lie
+// about either even in principle: the wire shape below has no field to lie
+// in (the meta part carries only version and kind).
+//
+// Streamed, never buffered to disk: r.MultipartReader() reads one part at a
+// time. ParseMultipartForm was ruled out on purpose — it spills whatever
+// doesn't fit its own in-memory threshold to a temp file under an
+// INDEPENDENT size limit, one that would sit alongside apiShipMaxBytes
+// rather than being governed by it. Each part is still read fully into
+// memory here, because release.Ship's own shape (map[string][]byte) leaves
+// no smaller option — the same buffering the CLI's own direct-ship path
+// already does with os.ReadFile — but the MaxBytesReader wrapped around the
+// whole request body below is what actually caps the total.
+func (s *Server) handleAPIShip(w http.ResponseWriter, r *http.Request) {
+ addr, acct, app, ok := s.apiAccountApp(w, r)
+ if !ok {
+ return
+ }
+ // Keyed on the TOKEN, not the address: ships are rare and heavy — bytes
+ // on the wire, an object written per artifact — so a token is the
+ // natural principal for an upload, unlike a write budget. Keying by
+ // address instead would make a laptop and a CI runner sharing one
+ // human's identity share one shipping budget between them, which is the
+ // limit doing less than it should, not more.
+ if !s.cli.ship.allow(hashToken(bearer(r)), time.Now()) {
+ jsonError(w, http.StatusTooManyRequests, "too many uploads — wait a while and try again")
+ return
+ }
+
+ r.Body = http.MaxBytesReader(w, r.Body, apiShipMaxBytes)
+ mr, err := r.MultipartReader()
+ if err != nil {
+ jsonError(w, http.StatusBadRequest, "expected a multipart/form-data body")
+ return
+ }
+
+ first, err := mr.NextPart()
+ if err != nil {
+ if shipMaxBytesRefused(w, err) {
+ return
+ }
+ jsonError(w, http.StatusBadRequest, "empty upload")
+ return
+ }
+ if first.FormName() != "meta" {
+ jsonError(w, http.StatusBadRequest, `the first part must be named "meta"`)
+ return
+ }
+ var meta struct {
+ Version string `json:"version"`
+ Kind string `json:"kind"`
+ }
+ if err := json.NewDecoder(io.LimitReader(first, 8<<10)).Decode(&meta); err != nil {
+ jsonError(w, http.StatusBadRequest, "malformed meta part")
+ return
+ }
+ ver := strings.TrimSpace(meta.Version)
+ if !release.ValidVersion(ver) || ver == "dev" {
+ jsonError(w, http.StatusBadRequest, "invalid version")
+ return
+ }
+ if meta.Kind != "binary" && meta.Kind != "static" {
+ jsonError(w, http.StatusBadRequest, `invalid kind (want "binary" or "static")`)
+ return
+ }
+
+ files := make(map[string][]byte)
+ artifacts := make(map[string]release.Artifact)
+ for {
+ part, err := mr.NextPart()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ if shipMaxBytesRefused(w, err) {
+ return
+ }
+ jsonError(w, http.StatusBadRequest, "malformed multipart body")
+ return
+ }
+ name := part.FormName()
+ if meta.Kind == "static" {
+ if name != "site.tar.gz" {
+ jsonError(w, http.StatusBadRequest, `a static release ships exactly one file part, named "site.tar.gz"`)
+ return
+ }
+ } else if !targetRe.MatchString(name) {
+ jsonError(w, http.StatusBadRequest, "invalid artifact name "+name)
+ return
+ }
+ b, err := io.ReadAll(part)
+ if err != nil {
+ if shipMaxBytesRefused(w, err) {
+ return
+ }
+ jsonError(w, http.StatusBadRequest, "could not read part "+name)
+ return
+ }
+ sum := sha256.Sum256(b)
+ files[name] = b
+ artifacts[name] = release.Artifact{
+ File: name,
+ SHA256: hex.EncodeToString(sum[:]),
+ Size: int64(len(b)),
+ }
+ }
+ if len(files) == 0 {
+ jsonError(w, http.StatusBadRequest, "no artifact parts in the upload")
+ return
+ }
+
+ m := release.Manifest{
+ Schema: 1,
+ App: app,
+ Version: ver,
+ Kind: meta.Kind,
+ BuiltBy: "console-api " + version.Build,
+ BuiltAt: time.Now().UTC().Format(time.RFC3339),
+ Artifacts: artifacts,
+ }
+ // Longer than the other CLI doors' 15-20s: this one moves real bytes —
+ // up to apiShipMaxBytes of them — into the bucket, not a handful of
+ // short strings.
+ ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute)
+ defer cancel()
+ if err := release.Ship(ctx, s.cfg.Releases, acct.Sqid, m, files); err != nil {
+ log.Printf("console: api ship %s %s: %v", app, ver, err)
+ // The store failed, not the caller: a 502, matching the other doors'
+ // convention that a bucket-side error is never described to the
+ // caller in bucket-side terms.
+ jsonError(w, http.StatusBadGateway, "could not store the release — try again")
+ return
+ }
+ s.audit(acct, app, addr, "ship", "shipped "+ver+" (cli)")
+
+ out := make(map[string]any, len(artifacts))
+ for name, a := range artifacts {
+ out[name] = map[string]any{"sha256": a.SHA256, "size": a.Size}
+ }
+ writeJSON(w, http.StatusCreated, map[string]any{
+ "result": "shipped", "version": ver, "artifacts": out,
+ })
+}
diff --git a/internal/console/apicli_test.go b/internal/console/apicli_test.go
index 23d5b114d..bb758ce1e 100644
--- a/internal/console/apicli_test.go
+++ b/internal/console/apicli_test.go
@@ -11,8 +11,12 @@ package console
import (
"bytes"
"context"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/hex"
"encoding/json"
"io"
+ "mime/multipart"
"net/http"
"strings"
"testing"
@@ -654,3 +658,255 @@ func TestAPIRollbackStableRefused(t *testing.T) {
t.Fatalf("stable channel after two refused rollbacks = %q, want def5678 (unchanged)", cp.Version)
}
}
+
+// ---- ship (apicli.go) -------------------------------------------------------
+//
+// `carlos ship`'s door: a multipart upload whose bytes are hashed and sized
+// HERE, server-side, from what actually arrived on the wire — never from a
+// field a caller could set, because there is no such field on this wire
+// shape (the meta part carries only version and kind).
+
+// shipPart is one part of a multipart ship request, in the order it should
+// be written — order matters, since the handler insists "meta" come first.
+type shipPart struct {
+ name string
+ body []byte
+}
+
+// metaPart builds the JSON "meta" part every ship request starts with.
+func metaPart(t *testing.T, version, kind string) shipPart {
+ t.Helper()
+ raw, err := json.Marshal(map[string]string{"version": version, "kind": kind})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return shipPart{name: "meta", body: raw}
+}
+
+// randomBytes is a stand-in artifact payload: real bytes, not a fixed
+// string, so a hash mismatch would actually be caught rather than a lucky
+// coincidence.
+func randomBytes(t *testing.T, n int) []byte {
+ t.Helper()
+ b := make([]byte, n)
+ if _, err := rand.Read(b); err != nil {
+ t.Fatal(err)
+ }
+ return b
+}
+
+// postShip POSTs a multipart/form-data ship request built from parts, in
+// the order given, and decodes whatever JSON comes back (a refusal that
+// never reaches the handler's own JSON responses — e.g. a body cut short by
+// MaxBytesReader — decodes to an empty map, which the caller only inspects
+// via the status code).
+func postShip(t *testing.T, u, bearer string, parts []shipPart) (int, map[string]any) {
+ t.Helper()
+ var buf bytes.Buffer
+ mw := multipart.NewWriter(&buf)
+ for _, p := range parts {
+ fw, err := mw.CreateFormFile(p.name, p.name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := fw.Write(p.body); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := mw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ req, err := http.NewRequest(http.MethodPost, u, &buf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set("Content-Type", mw.FormDataContentType())
+ if bearer != "" {
+ req.Header.Set("Authorization", "Bearer "+bearer)
+ }
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer res.Body.Close()
+ raw, err := io.ReadAll(res.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := map[string]any{}
+ _ = json.Unmarshal(raw, &out)
+ return res.StatusCode, out
+}
+
+// TestAPIShipHappy: a fresh binary release ships through the multipart
+// door, lands at the exact bucket key the direct-to-bucket path would use,
+// and the manifest the server writes carries hashes/sizes it computed
+// itself — matching what the artifact bytes actually hash to, and BuiltBy
+// naming the console API as the shipper.
+func TestAPIShipHappy(t *testing.T) {
+ fx := newConsoleFixture(t)
+ now := time.Now()
+ if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", now); err != nil {
+ t.Fatal(err)
+ }
+ token := cliToken(t, fx, "paul@keymail.dev", "code-ship-happy")
+ payload := randomBytes(t, 1024)
+
+ status, out := postShip(t, fx.ts.URL+"/api/cli/apps/bab/titogo/releases", token,
+ []shipPart{metaPart(t, "abc1234", "binary"), {name: "linux-arm64", body: payload}})
+ if status != http.StatusCreated {
+ t.Fatalf("ship = %d %v", status, out)
+ }
+ if str(t, out, "result") != "shipped" || str(t, out, "version") != "abc1234" {
+ t.Fatalf("ship body = %v", out)
+ }
+ sum := sha256.Sum256(payload)
+ wantSHA := hex.EncodeToString(sum[:])
+ arts, _ := out["artifacts"].(map[string]any)
+ arm64, _ := arts["linux-arm64"].(map[string]any)
+ if arm64 == nil {
+ t.Fatalf("ship artifacts = %v, want linux-arm64", arts)
+ }
+ if str(t, arm64, "sha256") != wantSHA {
+ t.Fatalf("artifact sha256 = %v, want %s (computed from the bytes sent)", arm64["sha256"], wantSHA)
+ }
+ if sz, _ := arm64["size"].(float64); int64(sz) != int64(len(payload)) {
+ t.Fatalf("artifact size = %v, want %d", arm64["size"], len(payload))
+ }
+
+ ctx := context.Background()
+ stored, _, err := fx.rel.Get(ctx, release.ArtifactKey(release.OpsAccount, "titogo", "abc1234", "linux-arm64"))
+ if err != nil {
+ t.Fatalf("stored artifact bytes: %v", err)
+ }
+ if !bytes.Equal(stored, payload) {
+ t.Fatalf("stored artifact bytes differ from what was shipped")
+ }
+ m, err := release.GetManifest(ctx, fx.rel, release.OpsAccount, "titogo", "abc1234")
+ if err != nil {
+ t.Fatalf("manifest: %v", err)
+ }
+ a, ok := m.Artifacts["linux-arm64"]
+ if !ok {
+ t.Fatalf("manifest artifacts = %v, want linux-arm64", m.Artifacts)
+ }
+ if a.SHA256 != wantSHA || a.Size != int64(len(payload)) {
+ t.Fatalf("manifest artifact = %+v, want sha256=%s size=%d", a, wantSHA, len(payload))
+ }
+ if !strings.HasPrefix(m.BuiltBy, "console-api ") {
+ t.Fatalf("manifest built_by = %q, want prefix %q", m.BuiltBy, "console-api ")
+ }
+
+ events, err := fx.st.Events(OpsAccountID, "titogo", 10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ found := false
+ for _, e := range events {
+ if e.Action == "ship" && strings.Contains(e.Detail, "(cli)") {
+ found = true
+ }
+ }
+ if !found {
+ t.Fatalf("events = %v, want a ship row whose detail names the CLI", events)
+ }
+}
+
+// TestAPIShipStatic: a static site release ships under the single
+// "site.tar.gz" artifact name the static kind requires.
+func TestAPIShipStatic(t *testing.T) {
+ fx := newConsoleFixture(t)
+ now := time.Now()
+ if _, err := fx.st.CreateAppIn(OpsAccountID, "website", "paul@keymail.dev", now); err != nil {
+ t.Fatal(err)
+ }
+ token := cliToken(t, fx, "paul@keymail.dev", "code-ship-static")
+ payload := randomBytes(t, 512)
+
+ status, out := postShip(t, fx.ts.URL+"/api/cli/apps/bab/website/releases", token,
+ []shipPart{metaPart(t, "def5678", "static"), {name: "site.tar.gz", body: payload}})
+ if status != http.StatusCreated {
+ t.Fatalf("ship static = %d %v", status, out)
+ }
+ arts, _ := out["artifacts"].(map[string]any)
+ if _, ok := arts["site.tar.gz"]; !ok {
+ t.Fatalf("ship static artifacts = %v, want site.tar.gz", arts)
+ }
+}
+
+// TestAPIShipRefusals: every malformed shape of a ship request is a 400,
+// and none of them reach release.Ship.
+func TestAPIShipRefusals(t *testing.T) {
+ fx := newConsoleFixture(t)
+ now := time.Now()
+ if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", now); err != nil {
+ t.Fatal(err)
+ }
+ token := cliToken(t, fx, "paul@keymail.dev", "code-ship-refusals")
+ payload := randomBytes(t, 128)
+
+ cases := []struct {
+ name string
+ parts []shipPart
+ }{
+ {"bad version chars", []shipPart{
+ metaPart(t, "../etc/passwd", "binary"), {name: "linux-arm64", body: payload}}},
+ {"bad kind", []shipPart{
+ metaPart(t, "abc1234", "weird"), {name: "linux-arm64", body: payload}}},
+ {"binary part not a target name", []shipPart{
+ metaPart(t, "abc1234", "binary"), {name: "not-a-target", body: payload}}},
+ {"static part not site.tar.gz", []shipPart{
+ metaPart(t, "abc1234", "static"), {name: "linux-arm64", body: payload}}},
+ {"no file parts", []shipPart{
+ metaPart(t, "abc1234", "binary")}},
+ {"meta not first part", []shipPart{
+ {name: "linux-arm64", body: payload}, metaPart(t, "abc1234", "binary")}},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ status, out := postShip(t, fx.ts.URL+"/api/cli/apps/bab/titogo/releases", token, c.parts)
+ if status != http.StatusBadRequest {
+ t.Fatalf("%s = %d %v, want 400", c.name, status, out)
+ }
+ })
+ }
+}
+
+// TestAPIShipTooBig: apiShipMaxBytes bounds the WHOLE request, not any one
+// part — lowered here to 4KiB, an 8KiB artifact part is refused with 413
+// before it is ever handed to release.Ship.
+func TestAPIShipTooBig(t *testing.T) {
+ old := apiShipMaxBytes
+ apiShipMaxBytes = 4 << 10
+ defer func() { apiShipMaxBytes = old }()
+
+ fx := newConsoleFixture(t)
+ now := time.Now()
+ if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", now); err != nil {
+ t.Fatal(err)
+ }
+ token := cliToken(t, fx, "paul@keymail.dev", "code-ship-toobig")
+ payload := randomBytes(t, 8<<10)
+
+ status, out := postShip(t, fx.ts.URL+"/api/cli/apps/bab/titogo/releases", token,
+ []shipPart{metaPart(t, "abc1234", "binary"), {name: "linux-arm64", body: payload}})
+ if status != http.StatusRequestEntityTooLarge {
+ t.Fatalf("oversized ship = %d %v, want 413", status, out)
+ }
+}
+
+// TestAPIShipMemberGate: a non-member token gets the identical 404 every
+// other CLI-API door gives an unknown or someone-else's account — apiAccountApp
+// runs before any multipart parsing, so a garbage body never matters here.
+func TestAPIShipMemberGate(t *testing.T) {
+ fx := newConsoleFixture(t)
+ token := cliToken(t, fx, "eve@keymail.dev", "code-ship-gate")
+
+ status, body := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/releases", map[string]string{}, token)
+ if status != http.StatusNotFound {
+ t.Fatalf("ship as non-member = %d, want 404", status)
+ }
+ if body != "404 page not found\n" {
+ t.Fatalf("ship as non-member body = %q, want the stdlib http.NotFound output", body)
+ }
+}
diff --git a/internal/console/authcli.go b/internal/console/authcli.go
index 8aa9b9fd8..2ebb76b69 100644
--- a/internal/console/authcli.go
+++ b/internal/console/authcli.go
@@ -62,6 +62,18 @@ const (
cliWriteBurst = 30
// cliPollInterval is the wait the console asks the CLI to keep, seconds.
cliPollInterval = 2
+
+ // cliShipWindow is ship's own budget window — an hour, not the minute
+ // every other CLI limiter uses: a ship is a deliberate release cut, not
+ // a poll or a login guess, so the window matches how often a human (or
+ // a CI job) actually does this.
+ cliShipWindow = time.Hour
+ // cliShipBurst bounds ship uploads per TOKEN (see handleAPIShip) within
+ // that hour. Ships are rare: a build-and-ship is a deliberate act run a
+ // handful of times a day at most, so ten in an hour is roomy for
+ // legitimate use while making a stolen token's usefulness for flooding
+ // storage negligible.
+ cliShipBurst = 10
)
// rateLimiter is a fixed-window counter, keyed by whatever the caller thinks
@@ -126,7 +138,7 @@ func (l *rateLimiter) allow(key string, now time.Time) bool {
// cliLimits is the set of limiters the CLI endpoints share, hung off the
// Server so a test gets a fresh set per fixture.
type cliLimits struct {
- start, poll, pollIP, guessAddr, guessIP, write *rateLimiter
+ start, poll, pollIP, guessAddr, guessIP, write, ship *rateLimiter
}
func newCLILimits() *cliLimits {
@@ -140,6 +152,10 @@ func newCLILimits() *cliLimits {
guessAddr: newRateLimiter(cliGuessBurst, cliRateWindow),
guessIP: newRateLimiter(cliGuessBurst, cliRateWindow),
write: newRateLimiter(cliWriteBurst, cliRateWindow),
+ // Ship gets its own budget, on its own window, keyed by token rather
+ // than address (see handleAPIShip) — an upload is a different shape
+ // of cost (bandwidth, storage) from the JSON mutations write covers.
+ ship: newRateLimiter(cliShipBurst, cliShipWindow),
}
}
@@ -161,6 +177,7 @@ func (s *Server) registerCLIRoutes() {
s.mux.HandleFunc("GET /api/cli/apps/{account}/{app}/channels", s.handleAPIChannels)
s.mux.HandleFunc("POST /api/cli/apps/{account}/{app}/promote", s.handleAPIPromote)
s.mux.HandleFunc("POST /api/cli/apps/{account}/{app}/rollback", s.handleAPIRollback)
+ s.mux.HandleFunc("POST /api/cli/apps/{account}/{app}/releases", s.handleAPIShip)
}
// clientIP is who the rate limiter counts against.