carlos / platform Public

console: per-tenant mail counts, caps and the automatic pause

Enforcement is reactive because SES has no per-identity rate limit — a cap
of 50 is a cap of about 50, and the code says so. Rates are zero on a day
with no sends rather than NaN, which would compare false against every
threshold and never trip. Rate thresholds wait for a send floor so one
bounce on a two-message morning cannot pause a healthy tenant. Nothing
auto-resumes: a pause that lifts at midnight lets an abusive tenant send its
cap again every day forever.
Paul Campbell pushed by paul@keymail.dev 36553eea0f170fb1706844530b991896e3e465c3 parent 7a97fb1
4 files changed, +497 −4
  • internal/console/mailcounts.go +193 −0
  • internal/console/mailcounts_test.go +229 −0
  • internal/console/sesevents.go +58 −4
  • internal/console/store.go +17 −0
diff --git a/internal/console/mailcounts.go b/internal/console/mailcounts.go
new file mode 100644
index 000000000..a6cfd9a34
--- /dev/null
+++ b/internal/console/mailcounts.go
@@ -0,0 +1,193 @@
+// Per-tenant send, bounce and complaint counts, and the limits they enforce
+// (tenant-email spec §9, §10).
+//
+// ENFORCEMENT IS REACTIVE. SES has no per-identity rate limit, so a cap can
+// only be applied by counting what has already been sent and then stopping
+// further sends. There are minutes of latency between the send and the
+// event, which means a cap of 50 is a cap of "about 50". That is stated
+// here rather than implied, because a caller who believes it is exact will
+// eventually be surprised by a bill or a complaint.
+package console
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "log"
+ "time"
+
+ "github.com/carlosframework/platform/internal/mail"
+)
+
+// MailCounts is one app's activity for one UTC day.
+type MailCounts struct {
+ Sends int
+ Bounces int
+ Complaints int
+ Rejects int
+}
+
+// BounceRate and ComplaintRate are zero on a day with no sends — not NaN.
+// A NaN compares false against every threshold, so a divide-by-zero here
+// would present as a limiter that silently never trips.
+func (c MailCounts) BounceRate() float64 {
+ if c.Sends == 0 {
+ return 0
+ }
+ return float64(c.Bounces) / float64(c.Sends)
+}
+
+func (c MailCounts) ComplaintRate() float64 {
+ if c.Sends == 0 {
+ return 0
+ }
+ return float64(c.Complaints) / float64(c.Sends)
+}
+
+// The thresholds SES reviews an account on. Applied per tenant so one
+// tenant's stale list cannot put every tenant under review.
+const (
+ BounceThreshold = 0.05
+ ComplaintThreshold = 0.001
+ // MinSendsForRates stops a single bounce on a two-message day reading
+ // as a 50% bounce rate and pausing a healthy tenant.
+ MinSendsForRates = 50
+)
+
+// TierCap is the daily send cap for a tier.
+//
+// An untiered or unrecognised account gets the FREE cap, never an unlimited
+// one. That default is what lets this ship before the billing work lands,
+// and the direction matters: falling back to unlimited would hand every
+// unknown account an uncapped estate.
+func TierCap(tier string) int {
+ switch tier {
+ case "pro":
+ return 10000
+ case "starter":
+ return 1000
+ default:
+ return 50
+ }
+}
+
+// mailCountColumn maps an mail.Event* type to the column RecordMailEvent
+// increments. false means "not one of the four counted types" — the same
+// tolerance mail.ParseEvent extends to a type it does not recognise: SES
+// adds event types, and refusing one here would need to fail the whole
+// ingest rather than just skip counting it.
+func mailCountColumn(evType string) (string, bool) {
+ switch evType {
+ case mail.EventSend:
+ return "sends", true
+ case mail.EventBounce:
+ return "bounces", true
+ case mail.EventComplaint:
+ return "complaints", true
+ case mail.EventReject:
+ return "rejects", true
+ default:
+ return "", false
+ }
+}
+
+// RecordMailEvent increments the one counter evType names for (acct, app,
+// day). An event type this function does not know (e.g. Delivery, or a
+// future SES type) is silently ignored — mirrored from mail.ParseEvent's own
+// tolerance of unknown types, for the same reason: SES adding a type must
+// never turn into a 500 that makes SNS retry forever.
+func (s *Store) RecordMailEvent(ctx context.Context, acct, app, day, evType string) error {
+ col, ok := mailCountColumn(evType)
+ if !ok {
+ return nil
+ }
+ q := fmt.Sprintf(`INSERT INTO mail_counts (account, app, day, %s)
+VALUES (?, ?, ?, 1)
+ON CONFLICT(account, app, day) DO UPDATE SET %s = %s + 1`, col, col, col)
+ _, err := s.db.ExecContext(ctx, q, acct, app, day)
+ return err
+}
+
+// MailCounts returns one app's counts for one UTC day. A day with no rows at
+// all is a legitimate zero, not an error.
+func (s *Store) MailCounts(ctx context.Context, acct, app, day string) (MailCounts, error) {
+ var c MailCounts
+ row := s.db.QueryRowContext(ctx, `SELECT sends, bounces, complaints, rejects
+FROM mail_counts WHERE account = ? AND app = ? AND day = ?`, acct, app, day)
+ err := row.Scan(&c.Sends, &c.Bounces, &c.Complaints, &c.Rejects)
+ if err == sql.ErrNoRows {
+ return MailCounts{}, nil
+ }
+ if err != nil {
+ return MailCounts{}, err
+ }
+ return c, nil
+}
+
+// enforceMailLimits pauses an app that has exceeded its daily cap or is past
+// either SES threshold, and reports why.
+//
+// The rate thresholds are applied only once MinSendsForRates messages have
+// gone out that day: without that floor, one bounce on a two-message morning
+// is a 50% bounce rate and a healthy tenant is paused before it has sent
+// anything real.
+//
+// A pause MUST be visible. It is written to the account's audit log with the
+// counts that caused it and surfaced on the app page — a silent pause
+// presents to the member as "email stopped working", which is the failure
+// this whole mechanism is supposed to prevent someone else's tenant causing.
+func (s *Server) enforceMailLimits(ctx context.Context, acct, app string, now time.Time) (bool, string, error) {
+ rec, _, err := mail.Get(ctx, s.cfg.Releases, acct, app)
+ if err != nil {
+ return false, "", err
+ }
+ // Nothing provisioned, or already paused: there is nothing to stop.
+ // Re-pausing would rewrite the record and re-audit on every event, and
+ // — more importantly — would lift a pause that a human has not decided
+ // to lift. See "NOTHING auto-resumes" below.
+ if rec == nil || rec.Provisioned == nil || !rec.Provisioned.Active {
+ return false, "", nil
+ }
+ day := now.UTC().Format("2006-01-02")
+ c, err := s.store.MailCounts(ctx, acct, app, day)
+ if err != nil {
+ return false, "", err
+ }
+ cap := TierCap("")
+ if rec.Grant != nil && rec.Grant.DailyCap > 0 {
+ cap = rec.Grant.DailyCap
+ }
+
+ var reason string
+ switch {
+ case c.Sends >= cap:
+ reason = fmt.Sprintf("daily cap %d reached (%d sent)", cap, c.Sends)
+ // The send floor is what stops one bounce on a two-message morning
+ // reading as a 50% bounce rate and pausing a healthy tenant.
+ case c.Sends >= MinSendsForRates && c.BounceRate() > BounceThreshold:
+ reason = fmt.Sprintf("bounce rate %.1f%% over %.0f%% (%d of %d)",
+ c.BounceRate()*100, BounceThreshold*100, c.Bounces, c.Sends)
+ case c.Sends >= MinSendsForRates && c.ComplaintRate() > ComplaintThreshold:
+ reason = fmt.Sprintf("complaint rate %.2f%% over %.1f%% (%d of %d)",
+ c.ComplaintRate()*100, ComplaintThreshold*100, c.Complaints, c.Sends)
+ default:
+ return false, "", nil
+ }
+
+ if _, err := mail.SetPaused(ctx, s.cfg.Releases, s.cfg.MailProvisioner, acct, app, true, now); err != nil {
+ return false, "", err
+ }
+ // Audit AFTER the write it announces, and never before — a pause the
+ // member cannot see is the failure §9 exists to prevent someone else's
+ // tenant causing.
+ if a, err := s.store.AccountBySqid(acct); err != nil {
+ log.Printf("console: mail pause audit: resolving account %s: %v", acct, err)
+ } else if a != nil {
+ s.audit(a, app, "ses", "mail.paused", fmt.Sprintf("%s: %s", app, reason))
+ }
+ return true, reason, nil
+}
+
+// NOTHING auto-resumes. A pause that lifted itself at midnight would let an
+// abusive tenant send its cap again every day forever, which is not a limit
+// — it is a schedule. Resume is a decision, taken through the door.
diff --git a/internal/console/mailcounts_test.go b/internal/console/mailcounts_test.go
new file mode 100644
index 000000000..c675be91c
--- /dev/null
+++ b/internal/console/mailcounts_test.go
@@ -0,0 +1,229 @@
+package console
+
+import (
+ "context"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/mail"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The two thresholds SES itself reviews an account on. They are per-tenant
+// here so one tenant's bad list cannot take the estate down (spec §9).
+func TestRatesAreComputedOverSends(t *testing.T) {
+ c := MailCounts{Sends: 1000, Bounces: 60, Complaints: 2}
+ if got := c.BounceRate(); got < 0.059 || got > 0.061 {
+ t.Fatalf("BounceRate = %v, want ~0.06", got)
+ }
+ if got := c.ComplaintRate(); got < 0.0019 || got > 0.0021 {
+ t.Fatalf("ComplaintRate = %v, want ~0.002", got)
+ }
+}
+
+// Zero sends must be zero rates, not a divide-by-zero or a NaN that
+// compares false against every threshold and silently never trips.
+func TestRatesWithNoSendsAreZeroNotNaN(t *testing.T) {
+ c := MailCounts{}
+ if c.BounceRate() != 0 || c.ComplaintRate() != 0 {
+ t.Fatalf("rates on an empty day = %v/%v, want 0/0", c.BounceRate(), c.ComplaintRate())
+ }
+}
+
+// An untiered account defaults to free rather than to unlimited. This is
+// what lets the primitive ship before the billing work on #226 lands
+// (spec §10) — and getting it backwards would hand every unknown account
+// an uncapped estate.
+func TestUntieredAccountsGetTheFreeCap(t *testing.T) {
+ if TierCap("") != TierCap("free") {
+ t.Fatal("an untiered account did not default to the free cap")
+ }
+ if TierCap("free") != 50 {
+ t.Fatalf("free cap = %d, want 50", TierCap("free"))
+ }
+ if TierCap("unknown-tier-name") != TierCap("free") {
+ t.Fatal("an unrecognised tier did not fall back to free")
+ }
+}
+
+// mailFixture seeds a declared, granted, provisioned app so enforce has
+// something to pause. Mirrors storeFixture in apistore_test.go.
+func mailFixture(t *testing.T) (*consoleFixture, time.Time) {
+ t.Helper()
+ fx := newConsoleFixture(t)
+ now := time.Date(2026, 8, 19, 12, 0, 0, 0, time.UTC)
+ fx.srv.cfg.MailProvisioner = mail.NewMemProvisioner("1")
+ if _, err := fx.st.CreateAppIn(OpsAccountID, "seapointish", "paul@keymail.dev", now); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := mail.Declare(context.Background(), fx.rel, release.OpsAccount, "seapointish",
+ "SEAPOINTISH_SMTP", "noreply@x.com", "paul@rslw.com", now); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := mail.ApplyGrant(context.Background(), fx.rel, release.OpsAccount, "seapointish",
+ []string{"x.com"}, 50, "tier:free", "tier", now); err != nil {
+ t.Fatal(err)
+ }
+ // Provision by hand: this test is about the limiter, not the walk.
+ rec, etag, _ := mail.Get(context.Background(), fx.rel, release.OpsAccount, "seapointish")
+ rec.Provisioned = &mail.Provisioned{IAMUser: "u", KeyID: "AKIA1", Active: true, At: now}
+ if _, err := mail.Put(context.Background(), fx.rel, release.OpsAccount, "seapointish", rec, etag); err != nil {
+ t.Fatal(err)
+ }
+ p := fx.srv.cfg.MailProvisioner.(*mail.MemProvisioner)
+ p.EnsureUser(context.Background(), "u", mail.UserPath, []string{"arn:aws:ses:eu-west-1:1:identity/x.com"}, []string{"x.com"})
+ return fx, now
+}
+
+func seedCounts(t *testing.T, fx *consoleFixture, day string, c MailCounts) {
+ t.Helper()
+ ctx := context.Background()
+ for i := 0; i < c.Sends; i++ {
+ if err := fx.st.RecordMailEvent(ctx, release.OpsAccount, "seapointish", day, mail.EventSend); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for i := 0; i < c.Bounces; i++ {
+ fx.st.RecordMailEvent(ctx, release.OpsAccount, "seapointish", day, mail.EventBounce)
+ }
+ for i := 0; i < c.Complaints; i++ {
+ fx.st.RecordMailEvent(ctx, release.OpsAccount, "seapointish", day, mail.EventComplaint)
+ }
+}
+
+func TestEnforcePausesOnCapBreach(t *testing.T) {
+ fx, now := mailFixture(t)
+ seedCounts(t, fx, now.UTC().Format("2006-01-02"), MailCounts{Sends: 50})
+ paused, reason, err := fx.srv.enforceMailLimits(context.Background(), release.OpsAccount, "seapointish", now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !paused {
+ t.Fatal("cap reached and nothing paused")
+ }
+ if !strings.Contains(reason, "daily cap 50") {
+ t.Fatalf("reason = %q, want it to name the cap", reason)
+ }
+ rec, _, _ := mail.Get(context.Background(), fx.rel, release.OpsAccount, "seapointish")
+ if rec.Provisioned.Active {
+ t.Fatal("record still says active")
+ }
+}
+
+// Below the send floor, a bad ratio is noise. One bounce on a two-message
+// morning is a 50% bounce rate and must NOT pause a healthy tenant.
+func TestEnforceIgnoresRatesBelowTheSendFloor(t *testing.T) {
+ fx, now := mailFixture(t)
+ seedCounts(t, fx, now.UTC().Format("2006-01-02"), MailCounts{Sends: 2, Bounces: 1})
+ paused, _, err := fx.srv.enforceMailLimits(context.Background(), release.OpsAccount, "seapointish", now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if paused {
+ t.Fatal("paused on a 2-message day — the send floor is not being applied")
+ }
+}
+
+func TestEnforcePausesOnBounceRate(t *testing.T) {
+ fx, now := mailFixture(t)
+ // Cap is 50, so raise it first: this test is about the RATE, and a
+ // cap breach would mask it.
+ mail.ApplyGrant(context.Background(), fx.rel, release.OpsAccount, "seapointish",
+ []string{"x.com"}, 100000, "operator", "paul@rslw.com", now)
+ seedCounts(t, fx, now.UTC().Format("2006-01-02"), MailCounts{Sends: 100, Bounces: 10})
+ paused, reason, err := fx.srv.enforceMailLimits(context.Background(), release.OpsAccount, "seapointish", now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !paused {
+ t.Fatal("a 10% bounce rate did not pause")
+ }
+ if !strings.Contains(reason, "bounce rate") {
+ t.Fatalf("reason = %q", reason)
+ }
+}
+
+// A pause must not lift itself. Re-running enforce on a paused app with a
+// clean slate must leave it paused — resume is a decision, not a timer.
+func TestEnforceNeverAutoResumes(t *testing.T) {
+ fx, now := mailFixture(t)
+ seedCounts(t, fx, now.UTC().Format("2006-01-02"), MailCounts{Sends: 50})
+ fx.srv.enforceMailLimits(context.Background(), release.OpsAccount, "seapointish", now)
+
+ next := now.AddDate(0, 0, 1) // a fresh day, no counts at all
+ paused, _, err := fx.srv.enforceMailLimits(context.Background(), release.OpsAccount, "seapointish", next)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if paused {
+ t.Fatal("enforce re-paused an already-paused app — it should have returned early")
+ }
+ rec, _, _ := mail.Get(context.Background(), fx.rel, release.OpsAccount, "seapointish")
+ if rec.Provisioned.Active {
+ t.Fatal("the pause lifted itself at midnight")
+ }
+}
+
+// The other half of "nothing auto-resumes": TestEnforceNeverAutoResumes above
+// re-checks on a day with zero counts, which an early-return guard and a
+// mutant that DROPS that guard both pass — a day with nothing to trip trips
+// nothing either way, so that test alone does not prove the guard exists.
+// This test keeps the cap breached (even further breached) on the SAME day
+// and re-runs enforce: only the early-return guard stops a second pass, a
+// second SetPaused call and a second audit line. (Mutation testing caught
+// this gap — see the task report.)
+func TestEnforceAlreadyPausedDoesNotReprocess(t *testing.T) {
+ fx, now := mailFixture(t)
+ day := now.UTC().Format("2006-01-02")
+ seedCounts(t, fx, day, MailCounts{Sends: 50})
+ if _, _, err := fx.srv.enforceMailLimits(context.Background(), release.OpsAccount, "seapointish", now); err != nil {
+ t.Fatal(err)
+ }
+
+ // More sends land on the SAME day, well past the cap — a re-evaluating
+ // enforce would trip again.
+ seedCounts(t, fx, day, MailCounts{Sends: 25})
+ paused, reason, err := fx.srv.enforceMailLimits(context.Background(), release.OpsAccount, "seapointish", now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if paused {
+ t.Fatalf("enforce re-processed an already-paused app (reason %q) — the early-return guard is not applied", reason)
+ }
+
+ entries, err := fx.st.Events(OpsAccountID, "seapointish", 10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pauses := 0
+ for _, e := range entries {
+ if strings.Contains(e.Action, "mail.paused") {
+ pauses++
+ }
+ }
+ if pauses != 1 {
+ t.Fatalf("got %d mail.paused audit entries, want exactly 1 (a second pass must not re-audit)", pauses)
+ }
+}
+
+// A pause nobody can see presents to the member as "email stopped working".
+// The real audit reader is Store.Events(accountID, app, limit) — see F7:
+// Store has no AuditEntries method, and s.audit(acct *Account, app, addr,
+// action, detail string) is what production code calls.
+func TestEnforceRecordsTheReasonInTheAuditLog(t *testing.T) {
+ fx, now := mailFixture(t)
+ seedCounts(t, fx, now.UTC().Format("2006-01-02"), MailCounts{Sends: 50})
+ fx.srv.enforceMailLimits(context.Background(), release.OpsAccount, "seapointish", now)
+
+ entries, err := fx.st.Events(OpsAccountID, "seapointish", 10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, e := range entries {
+ if strings.Contains(e.Action, "mail.paused") && strings.Contains(e.Detail, "daily cap 50") {
+ return
+ }
+ }
+ t.Fatalf("no audit entry naming the pause reason: %+v", entries)
+}
diff --git a/internal/console/sesevents.go b/internal/console/sesevents.go
index 76c352466..53925acfd 100644
--- a/internal/console/sesevents.go
+++ b/internal/console/sesevents.go
@@ -403,10 +403,7 @@ func (s *Server) handleSESEvents(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
return
}
- // Recording the per-tenant counts and enforcing the cap/pause is
- // Task 16 (mailcounts.go); this door's job ends at having proven
- // the event is authentic and parsed it.
- _ = ev
+ s.recordAndEnforceMail(r.Context(), ev)
w.WriteHeader(http.StatusOK)
default:
// An SES event type this deployment does not act on (SES has more
@@ -416,6 +413,63 @@ func (s *Server) handleSESEvents(w http.ResponseWriter, r *http.Request) {
}
}
+// recordAndEnforceMail resolves ev.Domain to the account(s)/app(s) that own
+// it and, for each, records the count and re-evaluates the cap and rate
+// thresholds (Task 16, mailcounts.go). Best-effort throughout: a lookup or
+// storage failure here is logged and swallowed, never turned into a non-2xx
+// — the caller has already committed to answering 200 for every recognised
+// notification, retried or not, and losing one count must not become an
+// unbounded SNS retry flood.
+//
+// Domain identity (mail.DomainRecord) is ACCOUNT-scoped, and there is no
+// reverse index from a bare domain to the account that owns it — so every
+// live console account is walked. Console deployments run a small,
+// bounded number of accounts; this is the same shape ListDomains' own
+// callers (mail.Teardown, mail.grantedDomains) already accept for a single
+// account, one level up.
+func (s *Server) recordAndEnforceMail(ctx context.Context, ev mail.Event) {
+ accts, err := s.store.Accounts()
+ if err != nil {
+ log.Printf("console: ses events: listing accounts for %s: %v", ev.Domain, err)
+ return
+ }
+ // The event's own timestamp decides which day's counters it lands in;
+ // a missing one (mail.ParseEvent leaves At zero when SES omits or
+ // mis-shapes mail.timestamp) falls back to now rather than bucketing
+ // into 0001-01-01. The SAME instant is then reused as enforceMailLimits'
+ // "now", so the cap/rate check reads the same day the count just landed
+ // in — using time.Now() there instead could evaluate a different UTC day
+ // than the one just written, right at midnight.
+ at := ev.At
+ if at.IsZero() {
+ at = time.Now()
+ }
+ day := at.UTC().Format("2006-01-02")
+ for _, a := range accts {
+ domains, err := mail.ListDomains(ctx, s.cfg.Releases, a.Sqid)
+ if err != nil {
+ log.Printf("console: ses events: listing domains for %s: %v", a.Sqid, err)
+ continue
+ }
+ for _, d := range domains {
+ if d.Domain != ev.Domain {
+ continue
+ }
+ for _, app := range d.Apps {
+ if err := s.store.RecordMailEvent(ctx, a.Sqid, app, day, ev.Type); err != nil {
+ log.Printf("console: ses events: recording %s for %s/%s: %v", ev.Type, a.Sqid, app, err)
+ continue
+ }
+ if _, reason, err := s.enforceMailLimits(ctx, a.Sqid, app, at); err != nil {
+ log.Printf("console: ses events: enforcing limits for %s/%s: %v", a.Sqid, app, err)
+ } else if reason != "" {
+ log.Printf("console: ses events: paused %s/%s: %s", a.Sqid, app, reason)
+ }
+ }
+ }
+ }
+}
+
// confirmSNSSubscription GETs the verified, host-checked SubscribeURL. It is
// reached only after the enclosing message's signature has verified AND
// SubscribeURL has passed validSigningCertURL — the signature proves AWS
diff --git a/internal/console/store.go b/internal/console/store.go
index daa7b4463..0d313da72 100644
--- a/internal/console/store.go
+++ b/internal/console/store.go
@@ -179,6 +179,23 @@ func (s *Store) migrate() error {
address TEXT NOT NULL,
firing INTEGER NOT NULL,
PRIMARY KEY (box, rule, address)
+);`,
+ // Per-tenant SES send/bounce/complaint/reject counts, one row per
+ // (account, app, day) — the abuse signal and the only source of a
+ // per-tenant send count (tenant-email spec §9, Task 16). Keyed by
+ // account (the sqid, matching mail.Record's own acct string), not
+ // account_id: this table is written from handleSESEvents, which
+ // resolves straight off mail.ListDomains and never looks up a
+ // console account row to get one.
+ `CREATE TABLE IF NOT EXISTS mail_counts (
+ account TEXT NOT NULL,
+ app TEXT NOT NULL,
+ day TEXT NOT NULL, -- UTC YYYY-MM-DD
+ sends INTEGER NOT NULL DEFAULT 0,
+ bounces INTEGER NOT NULL DEFAULT 0,
+ complaints INTEGER NOT NULL DEFAULT 0,
+ rejects INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY (account, app, day)
);`,
} {
if _, err := s.db.Exec(stmt); err != nil {