| 1 | // Per-tenant send, bounce and complaint counts, and the limits they enforce |
| 2 | // (tenant-email spec §9, §10). |
| 3 | // |
| 4 | // ENFORCEMENT IS REACTIVE. SES has no per-identity rate limit, so a cap can |
| 5 | // only be applied by counting what has already been sent and then stopping |
| 6 | // further sends. There are minutes of latency between the send and the |
| 7 | // event, which means a cap of 50 is a cap of "about 50". That is stated |
| 8 | // here rather than implied, because a caller who believes it is exact will |
| 9 | // eventually be surprised by a bill or a complaint. |
| 10 | package console |
| 11 | |
| 12 | import ( |
| 13 | "context" |
| 14 | "fmt" |
| 15 | "log" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/mail" |
| 19 | ) |
| 20 | |
| 21 | // MailCounts is one app's activity for one UTC day. |
| 22 | type MailCounts struct { |
| 23 | Sends int |
| 24 | Bounces int |
| 25 | Complaints int |
| 26 | Rejects int |
| 27 | // Deliveries is the denominator that makes a bounce count readable — |
| 28 | // "12 bounces" means something different against 50 delivered than |
| 29 | // against 50 sent and 12 delivered. Display-only, deliberately: see |
| 30 | // BounceRate below and the migration's own note. |
| 31 | Deliveries int |
| 32 | } |
| 33 | |
| 34 | // BounceRate and ComplaintRate are zero on a day with no sends — not NaN. |
| 35 | // A NaN compares false against every threshold, so a divide-by-zero here |
| 36 | // would present as a limiter that silently never trips. |
| 37 | func (c MailCounts) BounceRate() float64 { |
| 38 | if c.Sends == 0 { |
| 39 | return 0 |
| 40 | } |
| 41 | return float64(c.Bounces) / float64(c.Sends) |
| 42 | } |
| 43 | |
| 44 | func (c MailCounts) ComplaintRate() float64 { |
| 45 | if c.Sends == 0 { |
| 46 | return 0 |
| 47 | } |
| 48 | return float64(c.Complaints) / float64(c.Sends) |
| 49 | } |
| 50 | |
| 51 | // The thresholds SES reviews an account on. Applied per tenant so one |
| 52 | // tenant's stale list cannot put every tenant under review. |
| 53 | const ( |
| 54 | BounceThreshold = 0.05 |
| 55 | ComplaintThreshold = 0.001 |
| 56 | // MinSendsForRates stops a single bounce on a two-message day reading |
| 57 | // as a 50% bounce rate and pausing a healthy tenant. |
| 58 | MinSendsForRates = 50 |
| 59 | ) |
| 60 | |
| 61 | // TierCap is the daily send cap for a tier. |
| 62 | // |
| 63 | // An untiered or unrecognised account gets the FREE cap, never an unlimited |
| 64 | // one. That default is what lets this ship before the billing work lands, |
| 65 | // and the direction matters: falling back to unlimited would hand every |
| 66 | // unknown account an uncapped estate. |
| 67 | func TierCap(tier string) int { |
| 68 | switch tier { |
| 69 | case "pro": |
| 70 | return 10000 |
| 71 | case "starter": |
| 72 | return 1000 |
| 73 | default: |
| 74 | return 50 |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | // mailCountColumn maps an mail.Event* type to the column RecordMailEvent |
| 79 | // increments. false means "not one of the four counted types" — the same |
| 80 | // tolerance mail.ParseEvent extends to a type it does not recognise: SES |
| 81 | // adds event types, and refusing one here would need to fail the whole |
| 82 | // ingest rather than just skip counting it. |
| 83 | func mailCountColumn(evType string) (string, bool) { |
| 84 | switch evType { |
| 85 | case mail.EventSend: |
| 86 | return "sends", true |
| 87 | case mail.EventBounce: |
| 88 | return "bounces", true |
| 89 | case mail.EventComplaint: |
| 90 | return "complaints", true |
| 91 | case mail.EventReject: |
| 92 | return "rejects", true |
| 93 | case mail.EventDelivery: |
| 94 | return "deliveries", true |
| 95 | default: |
| 96 | return "", false |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | // RecordMailEvent increments the one counter evType names for (acct, app, |
| 101 | // sender, day). An event type this function does not know is silently |
| 102 | // ignored — mirrored from mail.ParseEvent's own tolerance of unknown types, |
| 103 | // for the same reason: SES adding a type must never turn into a 500 that |
| 104 | // makes SNS retry forever. |
| 105 | // |
| 106 | // Writes mail_sender_counts alone. mail_counts stays on disk, unwritten, |
| 107 | // until DropLegacyMailCounts removes it a release later. |
| 108 | func (s *Store) RecordMailEvent(ctx context.Context, acct, app, sender, day, evType string) error { |
| 109 | col, ok := mailCountColumn(evType) |
| 110 | if !ok { |
| 111 | return nil |
| 112 | } |
| 113 | if sender == "" { |
| 114 | sender = mail.SenderAppID |
| 115 | } |
| 116 | q := fmt.Sprintf(`INSERT INTO mail_sender_counts (account, app, sender, day, %s) |
| 117 | VALUES (?, ?, ?, ?, 1) |
| 118 | ON CONFLICT(account, app, sender, day) DO UPDATE SET %s = %s + 1`, col, col, col) |
| 119 | // A write of acct/app's row alone, under that app's permit |
| 120 | // (claim_mail_sql.go). |
| 121 | return s.withMailAppSQL(ctx, acct, app, "sql.RecordMailEvent", func(v *Store) error { |
| 122 | _, err := v.db.ExecContext(ctx, q, acct, app, sender, day) |
| 123 | return err |
| 124 | }) |
| 125 | } |
| 126 | |
| 127 | // MailCounts returns one sender's counts for one UTC day. An EMPTY sender |
| 128 | // reads every sender the app has, which is what the status reply's |
| 129 | // unchanged top-level figures mean and what a roll-up of an app in |
| 130 | // per-instance mode needs. A day with no rows at all is a legitimate zero. |
| 131 | func (s *Store) MailCounts(ctx context.Context, acct, app, sender, day string) (MailCounts, error) { |
| 132 | q := `SELECT COALESCE(SUM(sends), 0), COALESCE(SUM(bounces), 0), COALESCE(SUM(complaints), 0), |
| 133 | COALESCE(SUM(rejects), 0), COALESCE(SUM(deliveries), 0) |
| 134 | FROM mail_sender_counts WHERE account = ? AND app = ? AND day = ?` |
| 135 | args := []any{acct, app, day} |
| 136 | if sender != "" { |
| 137 | q += ` AND sender = ?` |
| 138 | args = append(args, sender) |
| 139 | } |
| 140 | var c MailCounts |
| 141 | // An aggregate always returns exactly one row, so there is no ErrNoRows |
| 142 | // branch to get wrong and a quiet day reads as zeroes by construction. |
| 143 | if err := s.db.QueryRowContext(ctx, q, args...).Scan( |
| 144 | &c.Sends, &c.Bounces, &c.Complaints, &c.Rejects, &c.Deliveries); err != nil { |
| 145 | return MailCounts{}, err |
| 146 | } |
| 147 | return c, nil |
| 148 | } |
| 149 | |
| 150 | // resumeStaleCapPause is the ONE place the day-roll resume check lives |
| 151 | // (R2, review follow-up). enforceMailLimits' only production caller is the |
| 152 | // SES event path (recordAndEnforceMail): a paused app's IAM key is |
| 153 | // deactivated, so it sends nothing, so no events ever arrive to run that |
| 154 | // check, so an app alone on its own domain saw its cap pause as permanent |
| 155 | // as before the cap/rate split — while the CLI now promises it clears |
| 156 | // automatically. The email-status READ path (handleAPIEmailStatus) is the |
| 157 | // other caller: a member running `carlos email status` is exactly the |
| 158 | // moment a rolled-over daily budget should clear, and it is a moment that |
| 159 | // happens without SES traffic. |
| 160 | // |
| 161 | // Both callers share this one function so they can never drift. |
| 162 | // |
| 163 | // Fail-safe by construction, matching "RATE AND MANUAL PAUSES NEVER |
| 164 | // AUTO-RESUME" below: resumes ONLY when sn is currently paused (Active == |
| 165 | // false), the pause is a CAP pause on a sender that is not PerInstance |
| 166 | // (PauseKind == mail.PauseKindCap — a RATE pause, a PER-INSTANCE pause or a |
| 167 | // kind-less manual/pre-existing pause, PauseKind == "", all fall through |
| 168 | // untouched), and the pause was applied on a UTC day STRICTLY EARLIER than |
| 169 | // now — same-day resumes would make the cap meaningless. |
| 170 | // |
| 171 | // Returns the freshly re-read sender when it resumes (sn's SetSenderPaused |
| 172 | // write moves Active, PauseKind and At, so callers must not patch the |
| 173 | // in-memory copy by hand) and whether a resume happened; sn is passed |
| 174 | // straight back, and resumed is false, in every other case, including |
| 175 | // sn == nil or sn.Provisioned == nil. |
| 176 | func (s *Server) resumeStaleCapPause(ctx context.Context, acct, app, sender string, sn *mail.Sender, now time.Time) (*mail.Sender, bool, error) { |
| 177 | if sn == nil || sn.Provisioned == nil || sn.Provisioned.Active { |
| 178 | return sn, false, nil |
| 179 | } |
| 180 | // A RATE pause, a MANUAL pause and a PER-INSTANCE pause all fall through |
| 181 | // untouched. The first two never auto-resume by design; the third must |
| 182 | // not, because resuming reactivates every key including the standalones |
| 183 | // and the app credential every instance still holds. |
| 184 | if sn.Provisioned.PauseKind != mail.PauseKindCap || sn.PerInstance { |
| 185 | return sn, false, nil |
| 186 | } |
| 187 | pausedDay := sn.Provisioned.At.UTC().Format("2006-01-02") |
| 188 | today := now.UTC().Format("2006-01-02") |
| 189 | if pausedDay >= today { |
| 190 | // Paused earlier TODAY (or a clock oddity puts the pause in the |
| 191 | // future) — not a day roll yet. |
| 192 | return sn, false, nil |
| 193 | } |
| 194 | if _, err := mail.SetSenderPaused(ctx, s.claimMailStore(), s.cfg.MailProvisioner, acct, app, sender, false, "", now); err != nil { |
| 195 | return nil, false, err |
| 196 | } |
| 197 | if a, err := s.store.AccountBySqid(acct); err != nil { |
| 198 | log.Printf("console: mail resume audit: resolving account %s: %v", acct, err) |
| 199 | } else if a != nil { |
| 200 | s.audit(a, app, "ses", "mail.resumed", fmt.Sprintf("%s: daily cap pause cleared on the UTC day roll", mailAuditSubject(app, sender))) |
| 201 | } |
| 202 | fresh, _, err := mail.GetSender(ctx, s.cfg.Releases, acct, app, sender) |
| 203 | if err != nil { |
| 204 | return nil, false, err |
| 205 | } |
| 206 | return fresh, true, nil |
| 207 | } |
| 208 | |
| 209 | // enforceMailLimits pauses a sender that has exceeded its daily cap or is |
| 210 | // past either SES threshold, and reports why. The cap and the pause are the |
| 211 | // sender's own, so one sender's breach leaves its siblings sending. |
| 212 | // |
| 213 | // The rate thresholds are applied only once MinSendsForRates messages have |
| 214 | // gone out that day: without that floor, one bounce on a two-message morning |
| 215 | // is a 50% bounce rate and a healthy tenant is paused before it has sent |
| 216 | // anything real. |
| 217 | // |
| 218 | // A pause MUST be visible. It is written to the account's audit log with the |
| 219 | // counts that caused it and surfaced on the app page — a silent pause |
| 220 | // presents to the member as "email stopped working", which is the failure |
| 221 | // this whole mechanism is supposed to prevent someone else's tenant causing. |
| 222 | func (s *Server) enforceMailLimits(ctx context.Context, acct, app, sender string, now time.Time) (bool, string, error) { |
| 223 | sn, _, err := mail.GetSender(ctx, s.cfg.Releases, acct, app, sender) |
| 224 | if err != nil { |
| 225 | return false, "", err |
| 226 | } |
| 227 | if sn == nil || sn.Provisioned == nil { |
| 228 | return false, "", nil |
| 229 | } |
| 230 | if !sn.Provisioned.Active { |
| 231 | fresh, resumed, err := s.resumeStaleCapPause(ctx, acct, app, sender, sn, now) |
| 232 | if err != nil { |
| 233 | return false, "", err |
| 234 | } |
| 235 | if !resumed { |
| 236 | return false, "", nil |
| 237 | } |
| 238 | // Re-read via resumeStaleCapPause: SetSenderPaused just wrote a |
| 239 | // fresh sender (Active, PauseKind and At all moved), and everything |
| 240 | // below reads sn fresh rather than patching the in-memory copy by |
| 241 | // hand. |
| 242 | sn = fresh |
| 243 | if sn == nil || sn.Provisioned == nil || !sn.Provisioned.Active { |
| 244 | return false, "", nil |
| 245 | } |
| 246 | } |
| 247 | day := now.UTC().Format("2006-01-02") |
| 248 | // The real sender, not "": MailCounts treats "" as every sender the app |
| 249 | // has, and this cap belongs to ONE sender — summing every sibling's |
| 250 | // traffic into it would let one sender's cap trip on another sender's |
| 251 | // mail, which is exactly what keying the cap to the sender is meant to |
| 252 | // stop. |
| 253 | c, err := s.store.MailCounts(ctx, acct, app, sender, day) |
| 254 | if err != nil { |
| 255 | return false, "", err |
| 256 | } |
| 257 | cap := TierCap("") |
| 258 | if sn.Grant != nil && sn.Grant.DailyCap > 0 { |
| 259 | cap = sn.Grant.DailyCap |
| 260 | } |
| 261 | |
| 262 | var reason, kind string |
| 263 | switch { |
| 264 | case c.Sends >= cap: |
| 265 | reason = fmt.Sprintf("daily cap %d reached (%d sent)", cap, c.Sends) |
| 266 | kind = mail.PauseKindCap |
| 267 | // The send floor is what stops one bounce on a two-message morning |
| 268 | // reading as a 50% bounce rate and pausing a healthy tenant. |
| 269 | case c.Sends >= MinSendsForRates && c.BounceRate() > BounceThreshold: |
| 270 | reason = fmt.Sprintf("bounce rate %.1f%% over %.0f%% (%d of %d)", |
| 271 | c.BounceRate()*100, BounceThreshold*100, c.Bounces, c.Sends) |
| 272 | kind = mail.PauseKindRate |
| 273 | case c.Sends >= MinSendsForRates && c.ComplaintRate() > ComplaintThreshold: |
| 274 | reason = fmt.Sprintf("complaint rate %.2f%% over %.1f%% (%d of %d)", |
| 275 | c.ComplaintRate()*100, ComplaintThreshold*100, c.Complaints, c.Sends) |
| 276 | kind = mail.PauseKindRate |
| 277 | default: |
| 278 | return false, "", nil |
| 279 | } |
| 280 | |
| 281 | if _, err := mail.SetSenderPaused(ctx, s.claimMailStore(), s.cfg.MailProvisioner, acct, app, sender, true, kind, now); err != nil { |
| 282 | return false, "", err |
| 283 | } |
| 284 | // Audit AFTER the write it announces, and never before — a pause the |
| 285 | // member cannot see is the failure §9 exists to prevent someone else's |
| 286 | // tenant causing. |
| 287 | if a, err := s.store.AccountBySqid(acct); err != nil { |
| 288 | log.Printf("console: mail pause audit: resolving account %s: %v", acct, err) |
| 289 | } else if a != nil { |
| 290 | s.audit(a, app, "ses", "mail.paused", fmt.Sprintf("%s: %s", mailAuditSubject(app, sender), reason)) |
| 291 | } |
| 292 | return true, reason, nil |
| 293 | } |
| 294 | |
| 295 | // mailAuditSubject names what a pause or resume audit line is about. These |
| 296 | // lines render on the app overview page, so they are member-facing copy |
| 297 | // that never went through this repo's copy-review gate for the "/sender" |
| 298 | // form. Every app has exactly one sender today (the app sender, mail. |
| 299 | // SenderAppID), so naming it on that line would add "/app" for zero |
| 300 | // information a member could use, and would be an unreviewed copy change. |
| 301 | // Only a sender OTHER than the app sender earns the "/sender" suffix, which |
| 302 | // keeps every live app's audit line byte-identical to what it already |
| 303 | // prints. |
| 304 | func mailAuditSubject(app, sender string) string { |
| 305 | if sender == mail.SenderAppID { |
| 306 | return app |
| 307 | } |
| 308 | return app + "/" + sender |
| 309 | } |
| 310 | |
| 311 | // RATE AND MANUAL PAUSES NEVER AUTO-RESUME. A RATE pause that lifted itself |
| 312 | // at midnight would let an abusive tenant breach SES's account-review |
| 313 | // thresholds again every day forever, which is not a limit — it is a |
| 314 | // schedule. A MANUAL (operator) pause is a human's deliberate decision, |
| 315 | // which only a human — through the door — gets to undo. Resume for either |
| 316 | // is a decision, taken through the door. |
| 317 | // |
| 318 | // A CAP pause is different (I3, review follow-up, amending the rule this |
| 319 | // comment used to state without exception): it is computed from PER-DAY |
| 320 | // counts, so "never auto-resumes" turned an ordinary tier limit into a |
| 321 | // permanent one — see the day-roll check above. |
| 322 | |
amadan