amadan
Merge branch 'mail-events-log' into main
Merge branch 'mail-events-log' into main (email events: keep what happened to each message, and carlos email events to read it)
13 files changed,
+1859
−21
docs/site/cli.md+17 −1internal/console/apimail.go+67 −0internal/console/apimail_test.go+71 −0internal/console/mailevents.go+249 −0internal/console/mailevents_test.go+479 −0internal/console/sesevents.go+90 −10internal/console/sesevents_test.go+1 −1internal/console/store.go+100 −2internal/mail/events.go+257 −6internal/mail/events_test.go+251 −1internal/mail/policy.go+46 −0mail_cli.go+177 −0mail_cli_test.go+54 −0
CI passed — run details
diff --git a/docs/site/cli.md b/docs/site/cli.md| index 0963b364d..79e2b6185 100644 |
| --- a/docs/site/cli.md |
| +++ b/docs/site/cli.md |
| @@ -866,7 +866,23 @@ it proves that mail leaves the building rather than that the console believes |
| it should. |
| `status` shows every domain's per-region verification state, the day's count |
| -against the cap, and whether sending is paused. `credentials create` mints a |
| +against the cap, and whether sending is paused. |
| + |
| +`events` is what happened to each message: delivered, bounced, complained |
| +about or delayed, newest first, with what the receiving server said — the |
| +line that tells you whether a bounce was a mistyped address or a full |
| +mailbox. `--message` narrows it to one message by the id SES gave it when it |
| +was sent, `--to` to one recipient, and `--since` to a window. Events are kept |
| +for 30 days. Where SES names the credential that sent a message, its events |
| +belong to that app alone, so two apps sharing a sending domain do not see |
| +each other's recipients; where it does not, they go to every app on the |
| +domain, as the day's counts always have. |
| + |
| +```sh |
| +carlos email events --app hello --since 24h |
| +carlos email events --app hello --to guest@example.com |
| +``` |
| + `credentials create` mints a |
| standalone SMTP credential for something not running on CARLOS — a laptop, a |
| cron box — printed once on that command's output and nowhere else. `rotate` |
| mints a fresh delivered credential and leaves the old one live until you run |
diff --git a/internal/console/apimail.go b/internal/console/apimail.go| index 59359f671..bcc6ed024 100644 |
| --- a/internal/console/apimail.go |
| +++ b/internal/console/apimail.go |
| @@ -55,6 +55,7 @@ import ( |
| "log" |
| "net/http" |
| "sort" |
| + "strconv" |
| "strings" |
| "time" |
| @@ -462,6 +463,10 @@ func (s *Server) mailWalkReply(ctx context.Context, w http.ResponseWriter, acct |
| // .../store is: the status reply names the IAM user, the key id and the |
| // verified domains — the estate, not just the app. |
| // |
| +// GET .../email/events is Admin too, and for a different reason: it names |
| +// the people the app wrote to. A credential that may read an app's routes |
| +// or deploy it has no need of its recipients' addresses. |
| +// |
| // The three CLOSED doors (svcrole.Set{} — closed to every role, admin |
| // included) are each closed for their own reason: |
| // |
| @@ -489,6 +494,7 @@ func (s *Server) mailWalkReply(ctx context.Context, w http.ResponseWriter, acct |
| func (s *Server) registerEmailRoutes() { |
| s.cliRoute("POST /api/cli/apps/{account}/{app}/email", svcrole.Allow(svcrole.Admin), s.handleAPIEmailDeclare) |
| s.cliRoute("GET /api/cli/apps/{account}/{app}/email", svcrole.Allow(svcrole.Admin), s.handleAPIEmailStatus) |
| + s.cliRoute("GET /api/cli/apps/{account}/{app}/email/events", svcrole.Allow(svcrole.Admin), s.handleAPIEmailEvents) |
| s.cliRoute("POST /api/cli/apps/{account}/{app}/email/domains", svcrole.Allow(svcrole.Admin), s.handleAPIEmailDomainEnsure) |
| s.cliRoute("POST /api/cli/apps/{account}/{app}/email/domains/{domain}/verify", svcrole.Allow(svcrole.Admin), s.handleAPIEmailDomainVerify) |
| s.cliRoute("POST /api/cli/apps/{account}/{app}/email/enable", svcrole.Allow(svcrole.Admin), s.handleAPIEmailEnable) |
| @@ -648,6 +654,67 @@ func (s *Server) handleAPIEmailStatus(w http.ResponseWriter, r *http.Request) { |
| writeJSON(w, http.StatusOK, out) |
| } |
| +// ---- events ---------------------------------------------------------------- |
| + |
| +// handleAPIEmailEvents answers `carlos email events`: what happened to the |
| +// messages the app sent — delivered, bounced, complained about, delayed — |
| +// newest first, from the per-message log (mailevents.go). Not rate limited, |
| +// like every read door. |
| +// |
| +// It deliberately does NOT require the email record to exist: an app whose |
| +// email was torn down still has thirty days of history, and "what happened |
| +// to the last thing we sent" is exactly the question after a teardown. Nor |
| +// does it need the bucket — the log is the console's own table. What it |
| +// keeps from the family is the provisioner gate: a deployment with no email |
| +// answers 404 here as on every other email door. |
| +// |
| +// Query parameters, all optional: since (RFC 3339), message (an SES message |
| +// id), to (an address), limit (1–1000, default 100). |
| +func (s *Server) handleAPIEmailEvents(w http.ResponseWriter, r *http.Request) { |
| + _, acct, app, ok := s.apiAccountApp(w, r) |
| + if !ok { |
| + return |
| + } |
| + if !s.mailProvisionerGate(w) { |
| + return |
| + } |
| + q := MailEventQuery{ |
| + MessageID: strings.TrimSpace(r.URL.Query().Get("message")), |
| + Recipient: strings.TrimSpace(r.URL.Query().Get("to")), |
| + } |
| + if raw := strings.TrimSpace(r.URL.Query().Get("since")); raw != "" { |
| + since, err := time.Parse(time.RFC3339, raw) |
| + if err != nil { |
| + jsonError(w, http.StatusBadRequest, "since must be an RFC 3339 time, like 2026-09-15T08:00:00Z") |
| + return |
| + } |
| + q.Since = since |
| + } |
| + if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" { |
| + n, err := strconv.Atoi(raw) |
| + if err != nil || n < 1 || n > mailEventsMaxLimit { |
| + jsonError(w, http.StatusBadRequest, fmt.Sprintf("limit must be a whole number from 1 to %d", mailEventsMaxLimit)) |
| + return |
| + } |
| + q.Limit = n |
| + } |
| + ctx, cancel := context.WithTimeout(r.Context(), mailTimeout) |
| + defer cancel() |
| + events, err := s.store.MailEvents(ctx, acct.Sqid, app, q, time.Now()) |
| + if err != nil { |
| + log.Printf("console: api email events %s/%s: %v", acct.Sqid, app, err) |
| + jsonError(w, http.StatusBadGateway, "could not read the email log — try again") |
| + return |
| + } |
| + if events == nil { |
| + events = []MailEvent{} |
| + } |
| + writeJSON(w, http.StatusOK, map[string]any{ |
| + "events": events, |
| + "retention_days": int(mailEventsRetention / (24 * time.Hour)), |
| + }) |
| +} |
| + |
| // ---- domains ---------------------------------------------------------------- |
| // handleAPIEmailDomainEnsure answers `carlos email domains add`: bring a |
diff --git a/internal/console/apimail_test.go b/internal/console/apimail_test.go| index 2322f56d5..e90bc365f 100644 |
| --- a/internal/console/apimail_test.go |
| +++ b/internal/console/apimail_test.go |
| @@ -14,6 +14,7 @@ import ( |
| "context" |
| "encoding/json" |
| "net/http" |
| + "net/url" |
| "strings" |
| "testing" |
| "time" |
| @@ -163,6 +164,7 @@ func TestEmailDoorsAre404ForNonMembers(t *testing.T) { |
| call func(base, bearer string) (int, string) |
| }{ |
| {"GET status", func(base, bearer string) (int, string) { return getRaw(t, base, bearer) }}, |
| + {"GET events", func(base, bearer string) (int, string) { return getRaw(t, base+"/events", bearer) }}, |
| {"POST declare", func(base, bearer string) (int, string) { |
| return postRaw(t, base, map[string]any{"from": "hello@example.com"}, bearer) |
| }}, |
| @@ -224,6 +226,7 @@ func TestEmailDoorsAre404WithNoProvisioner(t *testing.T) { |
| do func() (int, string) |
| }{ |
| {"GET status", func() (int, string) { return getRaw(t, emailURL(fx, "titogo"), token) }}, |
| + {"GET events", func() (int, string) { return getRaw(t, emailURL(fx, "titogo")+"/events", token) }}, |
| {"POST declare", func() (int, string) { |
| return postRaw(t, emailURL(fx, "titogo"), map[string]any{"from": "hello@example.com"}, token) |
| }}, |
| @@ -1083,3 +1086,71 @@ func TestAPIEmailDomainEnsureRefusesADomainClaimedByAnotherAccount(t *testing.T) |
| t.Fatalf("refusal promised a retry would help, but this refusal is permanent: %q", msg) |
| } |
| } |
| + |
| +// TestAPIEmailEventsReadsTheLog: the door `carlos email events` calls hands |
| +// back the app's per-message log, narrowed by what the query asks, and |
| +// refuses a query it cannot read rather than guessing at it. |
| +func TestAPIEmailEventsReadsTheLog(t *testing.T) { |
| + fx, _, _, token := emailArmed(t, "code-email-events") |
| + app := "titogo" |
| + ctx := context.Background() |
| + now := time.Now() |
| + for _, c := range []struct{ to, id, typ string }{ |
| + {"ann@example.com", "m-ann", mail.EventDelivery}, {"bob@example.com", "m-bob", mail.EventBounce}, |
| + } { |
| + ev := mail.Event{Type: c.typ, MessageID: c.id, At: now, Outcomes: []mail.Outcome{{Recipient: c.to}}} |
| + if err := fx.st.RecordMailMessageEvent(ctx, fx.sqid, app, "sns-"+c.id, ev, true, now); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + |
| + status, out := getJSON(t, emailURL(fx, app)+"/events", token) |
| + if status != http.StatusOK { |
| + t.Fatalf("status = %d %v", status, out) |
| + } |
| + events, _ := out["events"].([]any) |
| + if len(events) != 2 { |
| + t.Fatalf("events = %v, want both", out["events"]) |
| + } |
| + if days, _ := out["retention_days"].(float64); days != 30 { |
| + t.Errorf("retention_days = %v, want 30 — the reader should be told how far back it can look", out["retention_days"]) |
| + } |
| + |
| + status, out = getJSON(t, emailURL(fx, app)+"/events?to=BOB%40example.com", token) |
| + events, _ = out["events"].([]any) |
| + if status != http.StatusOK || len(events) != 1 { |
| + t.Fatalf("narrowed to bob: %d %v", status, out) |
| + } |
| + if first, _ := events[0].(map[string]any); first["message_id"] != "m-bob" || first["type"] != mail.EventBounce { |
| + t.Errorf("row = %v", first) |
| + } |
| + |
| + // An app with nothing in its log is an empty list, not a null. |
| + if status, body := getRaw(t, emailURL(fx, app)+"/events?message=nothing-by-this-id", token); status != http.StatusOK || |
| + !strings.Contains(body, `"events":[]`) { |
| + t.Errorf("an empty answer = %d %s", status, body) |
| + } |
| + |
| + // And the narrowing that is not about who: a window and a count. |
| + old := mail.Event{Type: mail.EventDelivery, MessageID: "m-old", At: now.Add(-48 * time.Hour), Outcomes: []mail.Outcome{{Recipient: "cat@example.com"}}} |
| + if err := fx.st.RecordMailMessageEvent(ctx, fx.sqid, app, "sns-m-old", old, true, now); err != nil { |
| + t.Fatal(err) |
| + } |
| + since := url.QueryEscape(now.Add(-24 * time.Hour).UTC().Format(time.RFC3339)) |
| + if status, out := getJSON(t, emailURL(fx, app)+"/events?since="+since, token); status != http.StatusOK { |
| + t.Fatalf("since: %d %v", status, out) |
| + } else if events, _ := out["events"].([]any); len(events) != 2 { |
| + t.Errorf("since the last day = %d events, want the two recent ones: %v", len(events), out["events"]) |
| + } |
| + if status, out := getJSON(t, emailURL(fx, app)+"/events?limit=1", token); status != http.StatusOK { |
| + t.Fatalf("limit: %d %v", status, out) |
| + } else if events, _ := out["events"].([]any); len(events) != 1 { |
| + t.Errorf("limit=1 gave %d events", len(events)) |
| + } |
| + |
| + for _, bad := range []string{"since=yesterday", "limit=0", "limit=100000", "limit=many"} { |
| + if status, body := getRaw(t, emailURL(fx, app)+"/events?"+bad, token); status != http.StatusBadRequest { |
| + t.Errorf("%s = %d %s, want 400", bad, status, body) |
| + } |
| + } |
| +} |
diff --git a/internal/console/mailevents.go b/internal/console/mailevents.go| new file mode 100644 |
| index 000000000..9a5777ebe |
| --- /dev/null |
| +++ b/internal/console/mailevents.go |
| @@ -0,0 +1,249 @@ |
| +package console |
| + |
| +// The per-message email log: what happened to each message an app sent, to |
| +// whom, and what the receiving server said. The counts in mailcounts.go are |
| +// the abuse signal and stay exactly as they were; this sits beside them and |
| +// is fed from the same verified SES notifications, one row per recipient |
| +// per notification. |
| + |
| +import ( |
| + "context" |
| + "database/sql" |
| + "strings" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/mail" |
| +) |
| + |
| +// mailEventsRetention is how long a row is kept. The log carries recipient |
| +// addresses — somebody's guests, customers, users — so it is a window, not |
| +// an archive: long enough to answer "did that email last week arrive", no |
| +// longer. |
| +const mailEventsRetention = 30 * 24 * time.Hour |
| + |
| +// mailEventsPruneEvery bounds how often the prune runs. It is one indexed |
| +// DELETE, but it would otherwise run once per notification, and an hour's |
| +// slack on a thirty-day window changes nothing anybody can see. |
| +const mailEventsPruneEvery = time.Hour |
| + |
| +// MailEvent is one row of the log. |
| +type MailEvent struct { |
| + MessageID string `json:"message_id"` |
| + Type string `json:"type"` |
| + Recipient string `json:"recipient"` |
| + Status string `json:"status,omitempty"` |
| + Diagnostic string `json:"diagnostic,omitempty"` |
| + Detail string `json:"detail,omitempty"` |
| + BySender bool `json:"by_sender"` |
| + At time.Time `json:"at"` |
| +} |
| + |
| +// RecordMailMessageEvent keeps one notification for one app: a row per |
| +// recipient it concerns. notice is SNS's MessageId for the notification, |
| +// and a notification SNS delivers twice writes nothing the second time. |
| +// bySender says whether SES itself named this app as the sender, rather |
| +// than the app being one of several that share the sending domain. |
| +func (s *Store) RecordMailMessageEvent(ctx context.Context, acct, app, notice string, ev mail.Event, bySender bool, now time.Time) error { |
| + at := ev.At |
| + if at.IsZero() { |
| + at = now |
| + } |
| + notice = mailNoticeKey(notice, ev, now) |
| + sender := 0 |
| + if bySender { |
| + sender = 1 |
| + } |
| + detail := ev.Detail.Summary() |
| + for _, o := range ev.Recipients() { |
| + if _, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO mail_events |
| + (account, app, notice, recipient, message_id, type, status, diagnostic, detail, by_sender, at, received_at) |
| +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, |
| + acct, app, notice, o.Recipient, ev.MessageID, ev.Type, o.Status, o.Diagnostic, detail, sender, |
| + at.Unix(), now.Unix()); err != nil { |
| + return err |
| + } |
| + } |
| + return s.pruneMailEvents(ctx, now) |
| +} |
| + |
| +// mailNoticeKey is the key a notification is kept under: SNS's own |
| +// MessageId, or — for a notification with no id of its own, which is nothing |
| +// SNS sends but the recording path is reachable directly — what it says, so |
| +// a repeat still collapses and two different notifications do not. |
| +func mailNoticeKey(notice string, ev mail.Event, now time.Time) string { |
| + if notice != "" { |
| + return notice |
| + } |
| + at := ev.At |
| + if at.IsZero() { |
| + at = now |
| + } |
| + var to []string |
| + for _, o := range ev.Recipients() { |
| + to = append(to, o.Recipient) |
| + } |
| + // Two bounces for one message — one per recipient, or a delay reported |
| + // twice — are two notifications, so what they are about is in the key. |
| + return ev.MessageID + "/" + ev.Type + "/" + at.UTC().Format(time.RFC3339Nano) + "/" + strings.Join(to, ",") |
| +} |
| + |
| +// mailAppTables are the tables keyed by an app's (account sqid, name) that |
| +// belong to that app alone: a name that is freed must not hand them on. |
| +var mailAppTables = []string{"mail_events"} |
| + |
| +// execer is what carryMailApp and forgetMailApp need from a transaction. |
| +type execer interface { |
| + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) |
| +} |
| + |
| +// carryMailApp moves an app's email history with the app, when a rename or |
| +// a move to another account changes the (sqid, name) it is keyed by. Rows |
| +// already at the destination belong to a purged predecessor of that name |
| +// and are dropped first, and only rows received since this app's own row was |
| +// created travel — the same rule RenameAppIn applies to audit rows, for the |
| +// same reason. Whatever is left under the old key is dropped. |
| +// |
| +// Across accounts, a row attributed only by its sending domain (by_sender = |
| +// 0) does not travel. Such a row was given to every app on the domain because |
| +// SES did not say which of them sent the message, so it may name another |
| +// app's recipient — and that app stays behind in the source account. Within |
| +// one account both kinds carry, since the same members could read both |
| +// before and after. |
| +func carryMailApp(ctx context.Context, tx execer, fromSqid, fromApp, toSqid, toApp string, createdAt int64) error { |
| + for _, table := range mailAppTables { |
| + if _, err := tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE account = ? AND app = ?`, toSqid, toApp); err != nil { |
| + return err |
| + } |
| + if table == "mail_events" && fromSqid != toSqid { |
| + if _, err := tx.ExecContext(ctx, `DELETE FROM mail_events WHERE account = ? AND app = ? AND by_sender = 0`, fromSqid, fromApp); err != nil { |
| + return err |
| + } |
| + } |
| + if _, err := tx.ExecContext(ctx, `UPDATE `+table+` SET account = ?, app = ? WHERE account = ? AND app = ? AND `+ |
| + mailAppTimeColumn(table)+` >= ?`, toSqid, toApp, fromSqid, fromApp, createdAt); err != nil { |
| + return err |
| + } |
| + if _, err := tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE account = ? AND app = ?`, fromSqid, fromApp); err != nil { |
| + return err |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +// forgetMailAccount drops everything an account's apps left in these tables, |
| +// when the account itself is reaped. Nothing can reach the rows afterwards — |
| +// the sqid is tombstoned and never reused — but they are recipients' |
| +// addresses, and a window is not an archive. |
| +func forgetMailAccount(ctx context.Context, tx execer, sqid string) error { |
| + for _, table := range mailAppTables { |
| + if _, err := tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE account = ?`, sqid); err != nil { |
| + return err |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +// forgetMailApp drops everything a purged app left in these tables, so a |
| +// later app of the same name starts with none of it. |
| +func forgetMailApp(ctx context.Context, tx execer, sqid, app string) error { |
| + for _, table := range mailAppTables { |
| + if _, err := tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE account = ? AND app = ?`, sqid, app); err != nil { |
| + return err |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +// mailAppTimeColumn is the column that says when a row came to be. |
| +func mailAppTimeColumn(table string) string { |
| + if table == "mail_events" { |
| + return "received_at" |
| + } |
| + return "created_at" |
| +} |
| + |
| +// pruneMailEvents trims the log to its retention window, at most once per |
| +// mailEventsPruneEvery. The stamp is taken before the DELETE runs, so two |
| +// notifications arriving together do not both prune. |
| +func (s *Store) pruneMailEvents(ctx context.Context, now time.Time) error { |
| + last := s.mailEventsPruned.Load() |
| + if last != 0 && now.Unix()-last < int64(mailEventsPruneEvery/time.Second) { |
| + return nil |
| + } |
| + if !s.mailEventsPruned.CompareAndSwap(last, now.Unix()) { |
| + return nil |
| + } |
| + if _, err := s.db.ExecContext(ctx, `DELETE FROM mail_events WHERE received_at < ?`, now.Add(-mailEventsRetention).Unix()); err != nil { |
| + // Not pruned, so not stamped: the next notification tries again |
| + // rather than waiting out the hour. |
| + s.mailEventsPruned.CompareAndSwap(now.Unix(), last) |
| + return err |
| + } |
| + return nil |
| +} |
| + |
| +// MailEventQuery narrows a read of the log. The zero value is the newest |
| +// mailEventsDefaultLimit rows of the whole window. |
| +type MailEventQuery struct { |
| + Since time.Time // only rows SES timestamped at or after this |
| + MessageID string // only this message |
| + Recipient string // only this address, case-insensitively |
| + Limit int |
| +} |
| + |
| +const ( |
| + mailEventsDefaultLimit = 100 |
| + mailEventsMaxLimit = 1000 |
| +) |
| + |
| +// MailEvents reads one app's log, newest first. A row older than the |
| +// retention window is never returned, whether or not the prune has reached |
| +// it yet — what a member is told is kept for thirty days has to be what |
| +// they can see. |
| +func (s *Store) MailEvents(ctx context.Context, acct, app string, q MailEventQuery, now time.Time) ([]MailEvent, error) { |
| + limit := q.Limit |
| + if limit <= 0 { |
| + limit = mailEventsDefaultLimit |
| + } |
| + if limit > mailEventsMaxLimit { |
| + limit = mailEventsMaxLimit |
| + } |
| + where := []string{"account = ?", "app = ?", "received_at >= ?"} |
| + args := []any{acct, app, now.Add(-mailEventsRetention).Unix()} |
| + if !q.Since.IsZero() { |
| + where = append(where, "at >= ?") |
| + args = append(args, q.Since.Unix()) |
| + } |
| + if q.MessageID != "" { |
| + where = append(where, "message_id = ?") |
| + args = append(args, q.MessageID) |
| + } |
| + if q.Recipient != "" { |
| + where = append(where, "recipient = ? COLLATE NOCASE") |
| + args = append(args, q.Recipient) |
| + } |
| + args = append(args, limit) |
| + rows, err := s.db.QueryContext(ctx, `SELECT message_id, type, recipient, status, diagnostic, detail, by_sender, at |
| +FROM mail_events WHERE `+strings.Join(where, " AND ")+` |
| +ORDER BY at DESC, received_at DESC, recipient LIMIT ?`, args...) |
| + if err != nil { |
| + return nil, err |
| + } |
| + defer rows.Close() |
| + var out []MailEvent |
| + for rows.Next() { |
| + var e MailEvent |
| + var at int64 |
| + var sender int |
| + if err := rows.Scan(&e.MessageID, &e.Type, &e.Recipient, &e.Status, &e.Diagnostic, &e.Detail, &sender, &at); err != nil { |
| + return nil, err |
| + } |
| + e.BySender = sender == 1 |
| + e.At = time.Unix(at, 0).UTC() |
| + out = append(out, e) |
| + } |
| + if err := rows.Err(); err != nil && err != sql.ErrNoRows { |
| + return nil, err |
| + } |
| + return out, nil |
| +} |
diff --git a/internal/console/mailevents_test.go b/internal/console/mailevents_test.go| new file mode 100644 |
| index 000000000..7b807f3a8 |
| --- /dev/null |
| +++ b/internal/console/mailevents_test.go |
| @@ -0,0 +1,479 @@ |
| +package console |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "net/http" |
| + "strconv" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/mail" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +func bounceTo(recipient, message string, at time.Time) mail.Event { |
| + return mail.Event{Type: mail.EventBounce, Domain: "lgc.li", MessageID: message, At: at, |
| + Outcomes: []mail.Outcome{{Recipient: recipient, Status: "5.1.1", Diagnostic: "550 5.1.1 no such user"}}, |
| + Detail: mail.EventDetail{BounceType: "Permanent", BounceSubType: "NoEmail"}} |
| +} |
| + |
| +// A bounce is kept as what it was: which message, to whom, and what the |
| +// receiving server answered — and a notification SNS delivers twice is |
| +// still one row. |
| +func TestTheMailLogKeepsEachRecipientsOutcomeOnce(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + now := time.Date(2026, 9, 15, 9, 0, 0, 0, time.UTC) |
| + ev := bounceTo("ann@example.com", "0102-ann", now.Add(-time.Minute)) |
| + |
| + for i := 0; i < 2; i++ { |
| + if err := fx.st.RecordMailMessageEvent(ctx, "beb", "jb-book", "sns-1", ev, true, now); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + got, err := fx.st.MailEvents(ctx, "beb", "jb-book", MailEventQuery{}, now) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got) != 1 { |
| + t.Fatalf("rows = %d, want 1 — SNS redelivered one notification: %+v", len(got), got) |
| + } |
| + e := got[0] |
| + if e.MessageID != "0102-ann" || e.Type != mail.EventBounce || e.Recipient != "ann@example.com" || |
| + e.Status != "5.1.1" || e.Detail != "Permanent/NoEmail" || !e.BySender { |
| + t.Errorf("row = %+v", e) |
| + } |
| + if !strings.Contains(e.Diagnostic, "no such user") { |
| + t.Errorf("the receiving server's answer was not kept: %q", e.Diagnostic) |
| + } |
| +} |
| + |
| +func TestTheMailLogFindsAMessageOrAnAddress(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + now := time.Date(2026, 9, 15, 9, 0, 0, 0, time.UTC) |
| + for i, c := range []struct{ to, id string }{ |
| + {"ann@example.com", "m-ann"}, {"bob@example.com", "m-bob"}, {"ann@example.com", "m-ann-2"}, |
| + } { |
| + ev := bounceTo(c.to, c.id, now.Add(-time.Duration(3-i)*time.Hour)) |
| + if err := fx.st.RecordMailMessageEvent(ctx, "beb", "jb-book", "sns-"+c.id, ev, true, now); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + for _, c := range []struct { |
| + name string |
| + q MailEventQuery |
| + want []string |
| + }{ |
| + {"everything, newest first", MailEventQuery{}, []string{"m-ann-2", "m-bob", "m-ann"}}, |
| + {"one message", MailEventQuery{MessageID: "m-bob"}, []string{"m-bob"}}, |
| + {"one address, however it is typed", MailEventQuery{Recipient: "ANN@Example.com"}, []string{"m-ann-2", "m-ann"}}, |
| + {"since", MailEventQuery{Since: now.Add(-150 * time.Minute)}, []string{"m-ann-2", "m-bob"}}, |
| + {"a limit", MailEventQuery{Limit: 1}, []string{"m-ann-2"}}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + got, err := fx.st.MailEvents(ctx, "beb", "jb-book", c.q, now) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var ids []string |
| + for _, e := range got { |
| + ids = append(ids, e.MessageID) |
| + } |
| + if strings.Join(ids, ",") != strings.Join(c.want, ",") { |
| + t.Errorf("got %v, want %v", ids, c.want) |
| + } |
| + }) |
| + } |
| + if other, _ := fx.st.MailEvents(ctx, "beb", "another-app", MailEventQuery{}, now); len(other) != 0 { |
| + t.Errorf("another app read this app's log: %+v", other) |
| + } |
| +} |
| + |
| +// The log holds recipients' addresses, so it is a window. A row past it is |
| +// neither shown nor kept. |
| +func TestTheMailLogForgetsAfterThirtyDays(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + then := time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC) |
| + if err := fx.st.RecordMailMessageEvent(ctx, "beb", "jb-book", "sns-old", bounceTo("ann@example.com", "m-old", then), true, then); err != nil { |
| + t.Fatal(err) |
| + } |
| + now := then.Add(mailEventsRetention + time.Hour) |
| + if got, _ := fx.st.MailEvents(ctx, "beb", "jb-book", MailEventQuery{}, now); len(got) != 0 { |
| + t.Errorf("a row past the window was shown: %+v", got) |
| + } |
| + // The next notification prunes it for good. |
| + if err := fx.st.RecordMailMessageEvent(ctx, "beb", "jb-book", "sns-new", bounceTo("bob@example.com", "m-new", now), true, now); err != nil { |
| + t.Fatal(err) |
| + } |
| + var n int |
| + if err := fx.st.db.QueryRow(`SELECT count(*) FROM mail_events WHERE message_id = 'm-old'`).Scan(&n); err != nil { |
| + t.Fatal(err) |
| + } |
| + if n != 0 { |
| + t.Errorf("the old row is still in the table") |
| + } |
| +} |
| + |
| +// Two apps can send from one domain. Their counts are attributed by the |
| +// domain, as they always were; the log is narrower wherever SES names the |
| +// sender, because a recipient's address is one app's business, not every |
| +// app's that happens to share a From domain. |
| +func TestTheMailLogGoesToTheAppSESSaysSentIt(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + now := time.Now() |
| + for _, app := range []string{"jb-book", "jb-site"} { |
| + if _, err := fx.st.CreateAppIn(OpsAccountID, app, "bill@tito.io", now); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + rec := &mail.DomainRecord{ |
| + Domain: "lgc.li", Kind: mail.KindCustom, Apps: []string{"jb-book", "jb-site"}, |
| + Regions: map[string]*mail.RegionState{"eu-west-1": { |
| + IdentityARN: "arn:aws:ses:eu-west-1:1:identity/lgc.li", MailFrom: "mail-eu-west-1.lgc.li", Status: mail.StatusVerified, |
| + }}, |
| + CreatedAt: now, |
| + } |
| + if _, err := mail.PutDomain(ctx, fx.rel, release.OpsAccount, rec, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + logOf := func(app string) []MailEvent { |
| + t.Helper() |
| + got, err := fx.st.MailEvents(ctx, release.OpsAccount, app, MailEventQuery{}, time.Now()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return got |
| + } |
| + |
| + named := bounceTo("ann@example.com", "m-named", now) |
| + named.Caller = mail.UserName(release.OpsAccount, "jb-book") |
| + fx.srv.recordAndEnforceMail(ctx, named, "sns-named") |
| + |
| + if got := logOf("jb-book"); len(got) != 1 || !got[0].BySender { |
| + t.Errorf("the app SES named should have the row, marked as its own: %+v", got) |
| + } |
| + if got := logOf("jb-site"); len(got) != 0 { |
| + t.Errorf("the other app on the domain was shown another app's recipient: %+v", got) |
| + } |
| + day := now.UTC().Format("2006-01-02") |
| + for _, app := range []string{"jb-book", "jb-site"} { |
| + if c, _ := fx.st.MailCounts(ctx, release.OpsAccount, app, day); c.Bounces != 1 { |
| + t.Errorf("%s bounces = %d — the counts' attribution is not this change's to narrow", app, c.Bounces) |
| + } |
| + } |
| + |
| + // SES naming one of CARLOS's own identities that is not a live app gives |
| + // the row to nobody — the name is the sender, and it is not here to be |
| + // told. It does NOT fall back to the domain's other apps. |
| + stray := bounceTo("eve@example.com", "m-stray", now) |
| + stray.Caller = mail.UserName(release.OpsAccount, "not-an-app") |
| + fx.srv.recordAndEnforceMail(ctx, stray, "sns-stray") |
| + for _, app := range []string{"jb-book", "jb-site"} { |
| + for _, e := range logOf(app) { |
| + if e.MessageID == "m-stray" { |
| + t.Errorf("%s was given a row SES attributed to another CARLOS app: %+v", app, e) |
| + } |
| + } |
| + } |
| + |
| + // SES naming nobody CARLOS minted falls back to the domain, and says so |
| + // on the row. |
| + for i, caller := range []string{"", "someone-else"} { |
| + ev := bounceTo("bob@example.com", "m-unnamed", now) |
| + ev.Caller = caller |
| + fx.srv.recordAndEnforceMail(ctx, ev, "sns-unnamed-"+string(rune('a'+i))) |
| + } |
| + for _, app := range []string{"jb-book", "jb-site"} { |
| + var unnamed []MailEvent |
| + for _, e := range logOf(app) { |
| + if e.MessageID == "m-unnamed" { |
| + unnamed = append(unnamed, e) |
| + } |
| + } |
| + if len(unnamed) != 2 { |
| + t.Errorf("%s: %d domain-attributed rows, want 2", app, len(unnamed)) |
| + } |
| + for _, e := range unnamed { |
| + if e.BySender { |
| + t.Errorf("%s: a row attributed by domain claims SES named the sender: %+v", app, e) |
| + } |
| + } |
| + } |
| +} |
| + |
| +// A trashed app is told nothing new, and a name that is freed — by a purge, |
| +// a rename or a move to another account — hands none of its email history |
| +// to whatever holds the name next. The history of an app that is renamed or |
| +// moved goes with it. |
| +func TestTheMailLogBelongsToTheAppNotItsName(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + now := time.Now() |
| + logOf := func(acct, app string) int { |
| + t.Helper() |
| + got, err := fx.st.MailEvents(ctx, acct, app, MailEventQuery{}, time.Now()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return len(got) |
| + } |
| + send := func(app, message string) { |
| + t.Helper() |
| + ev := bounceTo("ann@example.com", message, now) |
| + ev.Caller = mail.UserName(release.OpsAccount, app) |
| + fx.srv.recordAndEnforceMail(ctx, ev, "sns-"+message) |
| + } |
| + for _, app := range []string{"harbour", "renamed-from", "moving"} { |
| + if _, err := fx.st.CreateAppIn(OpsAccountID, app, "bill@tito.io", now.Add(-time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + |
| + // Trashed: nothing new is kept. |
| + send("harbour", "m-live") |
| + if err := fx.st.TrashAppRow(OpsAccountID, "harbour", now); err != nil { |
| + t.Fatal(err) |
| + } |
| + send("harbour", "m-trashed") |
| + if n := logOf(release.OpsAccount, "harbour"); n != 1 { |
| + t.Errorf("harbour has %d rows, want 1 — nothing arriving after the trash is kept", n) |
| + } |
| + |
| + // Purged: the name starts clean for the next app to claim it. |
| + if err := fx.st.PurgeAppRow(OpsAccountID, "harbour"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if n := logOf(release.OpsAccount, "harbour"); n != 0 { |
| + t.Errorf("a purged app's history is still there for its name: %d rows", n) |
| + } |
| + |
| + // Renamed: the history goes with the app, and the old name holds none. |
| + send("renamed-from", "m-before-rename") |
| + if err := fx.st.RenameAppIn(OpsAccountID, "renamed-from", "renamed-to"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if logOf(release.OpsAccount, "renamed-to") != 1 || logOf(release.OpsAccount, "renamed-from") != 0 { |
| + t.Errorf("after a rename: to=%d from=%d, want 1 and 0", |
| + logOf(release.OpsAccount, "renamed-to"), logOf(release.OpsAccount, "renamed-from")) |
| + } |
| + |
| + // Moved to another account: the same, keyed by the new account's sqid. |
| + send("moving", "m-before-move") |
| + if err := fx.st.CreateAccount(99, "dest", "Destination", "bill@tito.io", now); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := fx.st.MoveApp("moving", OpsAccountID, 99); err != nil { |
| + t.Fatal(err) |
| + } |
| + if logOf("dest", "moving") != 1 || logOf(release.OpsAccount, "moving") != 0 { |
| + t.Errorf("after a move: dest=%d ops=%d, want 1 and 0", logOf("dest", "moving"), logOf(release.OpsAccount, "moving")) |
| + } |
| +} |
| + |
| +// A bounce, a complaint or a delivery can arrive long after its message was |
| +// sent, and the rate limiter divides a day's bounces by that same day's |
| +// sends. So the counts file an event under the day it was SENT, whatever |
| +// day it arrives — a late bounce landing on the new day would read as a |
| +// bounce rate over a handful of sends and pause a healthy tenant. |
| +func TestALateBounceCountsAgainstTheDayItWasSent(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + now := time.Now() |
| + if _, err := fx.st.CreateAppIn(OpsAccountID, "harbour", "bill@tito.io", now); err != nil { |
| + t.Fatal(err) |
| + } |
| + rec := &mail.DomainRecord{ |
| + Domain: "harbour.example", Kind: mail.KindCustom, Apps: []string{"harbour"}, |
| + Regions: map[string]*mail.RegionState{"eu-west-1": { |
| + IdentityARN: "arn:aws:ses:eu-west-1:1:identity/harbour.example", Status: mail.StatusVerified, |
| + }}, |
| + CreatedAt: now, |
| + } |
| + if _, err := mail.PutDomain(ctx, fx.rel, release.OpsAccount, rec, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + sent := time.Date(2026, 9, 14, 23, 50, 0, 0, time.UTC) |
| + ev := bounceTo("ann@example.com", "m-late", sent.Add(15*time.Minute)) // bounces at 00:05 the next day |
| + ev.Domain, ev.SentAt = "harbour.example", sent |
| + fx.srv.recordAndEnforceMail(ctx, ev, "sns-late") |
| + |
| + for day, want := range map[string]int{"2026-09-14": 1, "2026-09-15": 0} { |
| + c, err := fx.st.MailCounts(ctx, release.OpsAccount, "harbour", day) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if c.Bounces != want { |
| + t.Errorf("bounces on %s = %d, want %d — counted against the day it arrived, not the day it was sent", day, c.Bounces, want) |
| + } |
| + } |
| +} |
| + |
| +// A row attributed only by a shared sending domain may name another app's |
| +// recipient. It stays in the account where that other app lives: a move to |
| +// another account carries only the rows SES said were the app's own. A |
| +// rename within the account carries both, since the same members could read |
| +// both before and after. |
| +func TestAMoveLeavesDomainAttributedRowsBehind(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + now := time.Now() |
| + for _, app := range []string{"moving", "renaming"} { |
| + if _, err := fx.st.CreateAppIn(OpsAccountID, app, "bill@tito.io", now.Add(-time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + for i, bySender := range []bool{true, false} { |
| + ev := bounceTo("ann@example.com", app+"-"+strconv.FormatBool(bySender), now) |
| + if err := fx.st.RecordMailMessageEvent(ctx, release.OpsAccount, app, "sns-"+app+strconv.Itoa(i), ev, bySender, now); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + } |
| + if err := fx.st.CreateAccount(99, "dest", "Destination", "bill@tito.io", now); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := fx.st.MoveApp("moving", OpsAccountID, 99); err != nil { |
| + t.Fatal(err) |
| + } |
| + moved, _ := fx.st.MailEvents(ctx, "dest", "moving", MailEventQuery{}, time.Now()) |
| + if len(moved) != 1 || !moved[0].BySender { |
| + t.Errorf("a move to another account carried %+v, want only the app's own row", moved) |
| + } |
| + if left, _ := fx.st.MailEvents(ctx, release.OpsAccount, "moving", MailEventQuery{}, time.Now()); len(left) != 0 { |
| + t.Errorf("rows were left under the old account's key: %+v", left) |
| + } |
| + |
| + if err := fx.st.RenameAppIn(OpsAccountID, "renaming", "renamed"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if renamed, _ := fx.st.MailEvents(ctx, release.OpsAccount, "renamed", MailEventQuery{}, time.Now()); len(renamed) != 2 { |
| + t.Errorf("a rename within the account carried %d rows, want both", len(renamed)) |
| + } |
| +} |
| + |
| +// A reaped account's email history goes with it. |
| +func TestReapingAnAccountForgetsItsMailLog(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + now := time.Now() |
| + if err := fx.st.CreateAccount(99, "reaped", "Going", "bill@tito.io", now); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := fx.st.RecordMailMessageEvent(ctx, "reaped", "gone", "sns-1", bounceTo("ann@example.com", "m", now), true, now); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := fx.st.ReapAccount(99, now); err != nil { |
| + t.Fatal(err) |
| + } |
| + var n int |
| + fx.st.db.QueryRow(`SELECT count(*) FROM mail_events WHERE account = 'reaped'`).Scan(&n) |
| + if n != 0 { |
| + t.Errorf("%d rows outlived the account", n) |
| + } |
| +} |
| + |
| +// A prune that fails is not stamped, so the next notification tries again |
| +// rather than waiting out the hour. |
| +func TestAFailedPruneIsTriedAgain(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx, cancel := context.WithCancel(context.Background()) |
| + cancel() |
| + if err := fx.st.pruneMailEvents(ctx, time.Now()); err == nil { |
| + t.Fatal("a prune with a cancelled context should fail") |
| + } |
| + if got := fx.st.mailEventsPruned.Load(); got != 0 { |
| + t.Errorf("the stamp was left at %d after a failed prune", got) |
| + } |
| +} |
| + |
| +// The log's domain fallback gives recipients' addresses only to a domain its |
| +// account has verified. The counts keep matching as they always did. |
| +func TestTheDomainFallbackNeedsAVerifiedDomain(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + now := time.Now() |
| + if _, err := fx.st.CreateAppIn(OpsAccountID, "squatter", "bill@tito.io", now); err != nil { |
| + t.Fatal(err) |
| + } |
| + rec := &mail.DomainRecord{ |
| + Domain: "mail-eu-west-1.lgc.li", Kind: mail.KindCustom, Apps: []string{"squatter"}, |
| + Regions: map[string]*mail.RegionState{"eu-west-1": { |
| + IdentityARN: "arn:aws:ses:eu-west-1:1:identity/mail-eu-west-1.lgc.li", Status: mail.StatusPending, |
| + }}, |
| + CreatedAt: now, |
| + } |
| + if _, err := mail.PutDomain(ctx, fx.rel, release.OpsAccount, rec, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + ev := bounceTo("ann@example.com", "m-unverified", now) |
| + ev.Domain, ev.SentAt = "mail-eu-west-1.lgc.li", now |
| + fx.srv.recordAndEnforceMail(ctx, ev, "sns-unverified") |
| + if got, _ := fx.st.MailEvents(ctx, release.OpsAccount, "squatter", MailEventQuery{}, time.Now()); len(got) != 0 { |
| + t.Errorf("an unverified domain was given recipients' addresses: %+v", got) |
| + } |
| + if c, _ := fx.st.MailCounts(ctx, release.OpsAccount, "squatter", now.UTC().Format("2006-01-02")); c.Bounces != 1 { |
| + t.Errorf("bounces = %d — the counts' matching is not this change's to narrow", c.Bounces) |
| + } |
| +} |
| + |
| +// End to end through the SNS door: a signed notification lands in the log |
| +// under SNS's own MessageId, with what the receiving server said, and a |
| +// redelivery of the same notification is still one row. |
| +func TestANotificationThroughTheDoorLandsInTheLog(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + fx.srv.cfg.MailDeployment.SNSTopicARN = "arn:aws:sns:eu-west-1:1:carlos-mail" |
| + fx.srv.snsVerify = func(_ snsEnvelope) error { return nil } |
| + ctx := context.Background() |
| + now := time.Now() |
| + if _, err := fx.st.CreateAppIn(OpsAccountID, "jb-book", "bill@tito.io", now); err != nil { |
| + t.Fatal(err) |
| + } |
| + rec := &mail.DomainRecord{ |
| + Domain: "lgc.li", Kind: mail.KindCustom, Apps: []string{"jb-book"}, |
| + Regions: map[string]*mail.RegionState{"eu-west-1": { |
| + IdentityARN: "arn:aws:ses:eu-west-1:1:identity/lgc.li", Status: mail.StatusVerified, |
| + }}, |
| + CreatedAt: now, |
| + } |
| + if _, err := mail.PutDomain(ctx, fx.rel, release.OpsAccount, rec, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + message, _ := json.Marshal(map[string]any{ |
| + "eventType": "Bounce", |
| + "mail": map[string]any{"source": "bookings@lgc.li", "messageId": "0102-door", "timestamp": now.UTC().Format(time.RFC3339), |
| + "tags": map[string][]string{"ses:caller-identity": {mail.UserName(release.OpsAccount, "jb-book")}}}, |
| + "bounce": map[string]any{"bounceType": "Permanent", "bounceSubType": "NoEmail", "timestamp": now.UTC().Format("2006-01-02T15:04:05.000Z"), |
| + "bouncedRecipients": []map[string]string{{"emailAddress": "ann@example.com", "status": "5.1.1", "diagnosticCode": "550 no such user"}}}, |
| + }) |
| + envelope, _ := json.Marshal(map[string]string{ |
| + "Type": "Notification", "MessageId": "sns-door-1", "TopicArn": "arn:aws:sns:eu-west-1:1:carlos-mail", |
| + "Message": string(message), "SigningCertURL": "https://sns.eu-west-1.amazonaws.com/x.pem", "Signature": "x", |
| + }) |
| + for i := 0; i < 2; i++ { |
| + resp := sesPost(t, fx, string(envelope)) |
| + resp.Body.Close() |
| + if resp.StatusCode != http.StatusOK { |
| + t.Fatalf("status = %d", resp.StatusCode) |
| + } |
| + } |
| + got, err := fx.st.MailEvents(ctx, release.OpsAccount, "jb-book", MailEventQuery{}, time.Now()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got) != 1 { |
| + t.Fatalf("rows = %d, want 1: %+v", len(got), got) |
| + } |
| + e := got[0] |
| + if e.MessageID != "0102-door" || e.Recipient != "ann@example.com" || e.Status != "5.1.1" || e.Detail != "Permanent/NoEmail" || !e.BySender { |
| + t.Errorf("row = %+v", e) |
| + } |
| + var notice string |
| + fx.st.db.QueryRow(`SELECT notice FROM mail_events WHERE message_id = '0102-door'`).Scan(¬ice) |
| + if notice != "sns-door-1" { |
| + t.Errorf("notice = %q, want SNS's own MessageId", notice) |
| + } |
| +} |
diff --git a/internal/console/sesevents.go b/internal/console/sesevents.go| index 05fabef6e..c0a4b83a0 100644 |
| --- a/internal/console/sesevents.go |
| +++ b/internal/console/sesevents.go |
| @@ -404,7 +404,11 @@ func (s *Server) handleSESEvents(w http.ResponseWriter, r *http.Request) { |
| w.WriteHeader(http.StatusOK) |
| return |
| } |
| - s.recordAndEnforceMail(r.Context(), ev) |
| + if ev.Incomplete != "" { |
| + // Counted all the same; only the log is missing something. |
| + log.Printf("console: ses events: %s notification %s read in part: %s", ev.Type, env.MessageID, ev.Incomplete) |
| + } |
| + s.recordAndEnforceMail(r.Context(), ev, env.MessageID) |
| w.WriteHeader(http.StatusOK) |
| default: |
| // An SES event type this deployment does not act on (SES has more |
| @@ -451,24 +455,51 @@ func domainMatchesEvent(d *mail.DomainRecord, domain string) bool { |
| // bounded number of accounts; this is the same shape ListDomains' own |
| // callers (mail.Teardown, mail.grantedDomains) already accept for a single |
| // account, one level up. |
| -func (s *Server) recordAndEnforceMail(ctx context.Context, ev mail.Event) { |
| +// |
| +// Each notification is also kept in the per-message log (mailevents.go), |
| +// under notice — SNS's own id for it. The log is attributed more narrowly |
| +// than the counts where SES allows: when the notification names a sending |
| +// identity CARLOS minted, the row goes to that app alone; only when it names |
| +// none does the row go to every app on the matching domain. The counts keep |
| +// their domain-wide attribution untouched — they enforce a cap, and changing |
| +// what they measure is not this change's business. |
| +func (s *Server) recordAndEnforceMail(ctx context.Context, ev mail.Event, notice string) { |
| accts, err := s.store.Accounts() |
| if err != nil { |
| log.Printf("console: ses events: listing accounts for %s: %v", ev.Domain, err) |
| return |
| } |
| - // The event's own timestamp decides which day's counters it lands in; |
| - // a missing one (mail.ParseEvent leaves At zero when SES omits or |
| - // mis-shapes mail.timestamp) falls back to now rather than bucketing |
| - // into 0001-01-01. The SAME instant is then reused as enforceMailLimits' |
| - // "now", so the cap/rate check reads the same day the count just landed |
| - // in — using time.Now() there instead could evaluate a different UTC day |
| - // than the one just written, right at midnight. |
| - at := ev.At |
| + // The SEND decides which day's counters an event lands in — never the |
| + // bounce, the complaint or the delivery, which can arrive hours or days |
| + // later. The rate limiter divides a day's bounces and complaints by that |
| + // same day's sends, so an event must count against the day its message |
| + // was sent: a bounce at 00:05 for a message sent at 23:50 counted on the |
| + // new day reads as a bounce rate over a handful of sends, and pauses a |
| + // healthy tenant. (ev.At is the event's own time, for the log; ev.SentAt |
| + // is mail.timestamp.) A missing send time (mail.ParseEvent leaves it |
| + // zero when SES omits or mis-shapes it) falls back to now rather than |
| + // bucketing into 0001-01-01. The SAME instant is then reused as |
| + // enforceMailLimits' "now", so the cap/rate check reads the same day the |
| + // count just landed in — using time.Now() there instead could evaluate a |
| + // different UTC day than the one just written, right at midnight. |
| + at := ev.SentAt |
| if at.IsZero() { |
| at = time.Now() |
| } |
| day := at.UTC().Format("2006-01-02") |
| + now := time.Now() |
| + key := mailNoticeKey(notice, ev, now) |
| + |
| + // The per-message log. When SES names one of CARLOS's own sending |
| + // identities, that app — and nobody else — gets the row, whatever domain |
| + // the message went out on: the credential is minted per app, so the name |
| + // is the sender. Only when SES names nobody CARLOS minted does the row |
| + // fall back to every app on the matching domain, below. |
| + callerAcct, callerApp, named := mail.AppOfCaller(ev.Caller) |
| + if named { |
| + s.recordMailMessage(ctx, callerAcct, callerApp, key, ev, true, now) |
| + } |
| + |
| for _, a := range accts { |
| domains, err := mail.ListDomains(ctx, s.cfg.Releases, a.Sqid) |
| if err != nil { |
| @@ -479,7 +510,16 @@ func (s *Server) recordAndEnforceMail(ctx context.Context, ev mail.Event) { |
| if !domainMatchesEvent(d, ev.Domain) { |
| continue |
| } |
| + // The log's fallback asks more of the match than the counts do: |
| + // only a domain this account actually verified gets recipients' |
| + // addresses. The domain claim refuses a second account the same |
| + // name, but not another account's MAIL FROM host claimed as a |
| + // domain of its own and never verified. |
| + logHere := !named && domainVerifiedForEvent(d, ev.Domain) |
| for _, app := range d.Apps { |
| + if logHere { |
| + s.recordMailMessage(ctx, a.Sqid, app, key, ev, false, now) |
| + } |
| if err := s.store.RecordMailEvent(ctx, a.Sqid, app, day, ev.Type); err != nil { |
| log.Printf("console: ses events: recording %s for %s/%s: %v", ev.Type, a.Sqid, app, err) |
| continue |
| @@ -494,6 +534,46 @@ func (s *Server) recordAndEnforceMail(ctx context.Context, ev mail.Event) { |
| } |
| } |
| +// domainVerifiedForEvent is domainMatchesEvent restricted to what this |
| +// record has proved: the identity domain in a region SES verified it in, or |
| +// the MAIL FROM host of a verified region. |
| +func domainVerifiedForEvent(d *mail.DomainRecord, domain string) bool { |
| + for _, st := range d.Regions { |
| + if st == nil || st.Status != mail.StatusVerified { |
| + continue |
| + } |
| + if d.Domain == domain || (st.MailFrom != "" && st.MailFrom == domain) { |
| + return true |
| + } |
| + } |
| + return false |
| +} |
| + |
| +// recordMailMessage keeps one notification in one app's log, if that app is |
| +// live. A trashed app keeps nothing new: its name is not a member-facing app |
| +// any more, and what it is sent is not somebody else's to read after a |
| +// restore or a claim. Best effort, like the rest of the ingest: a failure |
| +// is logged and costs neither the count nor the 200. |
| +func (s *Server) recordMailMessage(ctx context.Context, acct, app, key string, ev mail.Event, bySender bool, now time.Time) { |
| + if !s.mailAppLive(acct, app) { |
| + return |
| + } |
| + if err := s.store.RecordMailMessageEvent(ctx, acct, app, key, ev, bySender, now); err != nil { |
| + log.Printf("console: ses events: logging %s for %s/%s: %v", ev.Type, acct, app, err) |
| + } |
| +} |
| + |
| +// mailAppLive reports whether acct (a sqid) holds app as a live, untrashed |
| +// app right now. |
| +func (s *Server) mailAppLive(acct, app string) bool { |
| + a, err := s.store.AccountBySqid(acct) |
| + if err != nil || a == nil { |
| + return false |
| + } |
| + ok, err := s.store.HasAppIn(a.ID, app) |
| + return err == nil && ok |
| +} |
| + |
| // confirmSNSSubscription GETs the verified, host-checked SubscribeURL. It is |
| // reached only after the enclosing message's signature has verified AND |
| // SubscribeURL has passed validSigningCertURL — the signature proves AWS |
diff --git a/internal/console/sesevents_test.go b/internal/console/sesevents_test.go| index d02c32a5f..abb90406e 100644 |
| --- a/internal/console/sesevents_test.go |
| +++ b/internal/console/sesevents_test.go |
| @@ -772,7 +772,7 @@ func TestRecordAndEnforceMailAttributesTheRewrittenMailFromEnvelope(t *testing.T |
| } |
| ev := mail.Event{Type: mail.EventSend, Domain: "mail-eu-west-1.harbourmail.bag.oncarlos.com", At: now} |
| - fx.srv.recordAndEnforceMail(ctx, ev) |
| + fx.srv.recordAndEnforceMail(ctx, ev, "") |
| day := now.UTC().Format("2006-01-02") |
| counts, err := fx.srv.store.MailCounts(ctx, release.OpsAccount, "harbourmail", day) |
diff --git a/internal/console/store.go b/internal/console/store.go| index a1099538a..726435455 100644 |
| --- a/internal/console/store.go |
| +++ b/internal/console/store.go |
| @@ -6,6 +6,7 @@ |
| package console |
| import ( |
| + "context" |
| "crypto/ecdsa" |
| "crypto/elliptic" |
| "crypto/rand" |
| @@ -19,6 +20,7 @@ import ( |
| "os" |
| "path/filepath" |
| "strings" |
| + "sync/atomic" |
| "time" |
| _ "modernc.org/sqlite" |
| @@ -37,6 +39,12 @@ const ( |
| // deployment bucket. |
| type Store struct { |
| db *sql.DB |
| + |
| + // mailEventsPruned is when mail_events was last trimmed to its |
| + // retention window, as unix seconds (mailevents.go). Zero means not yet |
| + // this process, which is what makes the first event after a restart |
| + // prune. |
| + mailEventsPruned atomic.Int64 |
| } |
| // Open opens (creating if needed) the console DB at path. |
| @@ -197,6 +205,42 @@ func (s *Store) migrate() error { |
| rejects INTEGER NOT NULL DEFAULT 0, |
| PRIMARY KEY (account, app, day) |
| );`, |
| + // What happened to each message, per (account, app) — one row per |
| + // recipient per SES notification (mailevents.go). The counts above |
| + // say how many bounced today; this says which message, to whom, and |
| + // what the receiving server answered, so that a member can find out |
| + // and an app can be told. Keyed like mail_counts, by the account |
| + // sqid, for the same reason. |
| + // |
| + // notice is SNS's own MessageId for the notification: SNS delivers |
| + // at least once, and a notification delivered twice is one row, not |
| + // two. by_sender records HOW the row was attributed — 1 when SES |
| + // named this app's own sending identity, 0 when the sending domain |
| + // was all there was to go on and every app sharing it got a copy. |
| + // It is kept because the difference matters to anything that hands |
| + // a row on: only a row SES attributed to the app itself is that |
| + // app's to be told about. |
| + // |
| + // Recipient addresses are personal data, so the table is bounded in |
| + // time (mailEventsRetention) rather than kept for ever. |
| + `CREATE TABLE IF NOT EXISTS mail_events ( |
| + account TEXT NOT NULL, |
| + app TEXT NOT NULL, |
| + notice TEXT NOT NULL, |
| + recipient TEXT NOT NULL, |
| + message_id TEXT NOT NULL, |
| + type TEXT NOT NULL, |
| + status TEXT NOT NULL DEFAULT '', |
| + diagnostic TEXT NOT NULL DEFAULT '', |
| + detail TEXT NOT NULL DEFAULT '', |
| + by_sender INTEGER NOT NULL DEFAULT 0, |
| + at INTEGER NOT NULL, |
| + received_at INTEGER NOT NULL, |
| + PRIMARY KEY (account, app, notice, recipient) |
| +);`, |
| + `CREATE INDEX IF NOT EXISTS mail_events_by_time ON mail_events (account, app, at);`, |
| + `CREATE INDEX IF NOT EXISTS mail_events_by_message ON mail_events (account, app, message_id);`, |
| + `CREATE INDEX IF NOT EXISTS mail_events_by_age ON mail_events (received_at);`, |
| } { |
| if _, err := s.db.Exec(stmt); err != nil { |
| return err |
| @@ -1146,6 +1190,16 @@ func (s *Store) ReapAccount(accountID int64, now time.Time) error { |
| return err |
| } |
| } |
| + // Email history and settings are keyed by sqid (mailevents.go). |
| + var sqid string |
| + switch err := tx.QueryRow(`SELECT sqid FROM accounts WHERE id = ?`, accountID).Scan(&sqid); { |
| + case err == nil: |
| + if err := forgetMailAccount(context.Background(), tx, sqid); err != nil { |
| + return err |
| + } |
| + case err != sql.ErrNoRows: |
| + return err |
| + } |
| if _, err := tx.Exec(`UPDATE users SET last_account = 0 WHERE last_account = ?`, accountID); err != nil { |
| return err |
| } |
| @@ -1311,8 +1365,29 @@ func (s *Store) RestoreAppRow(accountID int64, app string) error { |
| // auto-purge pass this feeds must never be the thing that frees a serving |
| // app's name out from under it. |
| func (s *Store) PurgeAppRow(accountID int64, app string) error { |
| - _, err := s.db.Exec(`DELETE FROM apps WHERE account_id = ? AND app = ? AND deleted_at != 0`, accountID, app) |
| - return err |
| + ctx := context.Background() |
| + tx, err := s.db.BeginTx(ctx, nil) |
| + if err != nil { |
| + return err |
| + } |
| + defer tx.Rollback() |
| + res, err := tx.ExecContext(ctx, `DELETE FROM apps WHERE account_id = ? AND app = ? AND deleted_at != 0`, accountID, app) |
| + if err != nil { |
| + return err |
| + } |
| + if n, _ := res.RowsAffected(); n > 0 { |
| + // The name is free from here, so nothing keyed by it may outlive the |
| + // row (mailevents.go): a later app of the same name starts clean. |
| + var sqid string |
| + if err := tx.QueryRowContext(ctx, `SELECT sqid FROM accounts WHERE id = ?`, accountID).Scan(&sqid); err == nil { |
| + if err := forgetMailApp(ctx, tx, sqid, app); err != nil { |
| + return err |
| + } |
| + } else if err != sql.ErrNoRows { |
| + return err |
| + } |
| + } |
| + return tx.Commit() |
| } |
| // TrashedApp is one app in an account's Trash: past-tense on purpose, since a |
| @@ -1491,6 +1566,10 @@ LIMIT 1`, cliKindService, to, cliKindService, from, app).Scan(&collidingName) |
| if err == nil { |
| return fmt.Errorf("console: the destination account already holds a live service credential named %q — rename or revoke it first", collidingName) |
| } |
| + var createdAt int64 |
| + if err := tx.QueryRow(`SELECT created_at FROM apps WHERE app = ? AND account_id = ?`, app, from).Scan(&createdAt); err != nil { |
| + return err |
| + } |
| if _, err := tx.Exec(`UPDATE apps SET account_id = ? WHERE app = ? AND account_id = ?`, |
| to, app, from); err != nil { |
| return err |
| @@ -1499,6 +1578,18 @@ LIMIT 1`, cliKindService, to, cliKindService, from, app).Scan(&collidingName) |
| to, cliKindApp, cliKindService, from, app); err != nil { |
| return err |
| } |
| + // Email history is keyed by the account's sqid, not its id, so it is |
| + // carried by sqid (mailevents.go). |
| + var fromSqid, toSqid string |
| + if err := tx.QueryRow(`SELECT sqid FROM accounts WHERE id = ?`, from).Scan(&fromSqid); err != nil { |
| + return err |
| + } |
| + if err := tx.QueryRow(`SELECT sqid FROM accounts WHERE id = ?`, to).Scan(&toSqid); err != nil { |
| + return err |
| + } |
| + if err := carryMailApp(context.Background(), tx, fromSqid, app, toSqid, app, createdAt); err != nil { |
| + return err |
| + } |
| return tx.Commit() |
| } |
| @@ -1573,6 +1664,13 @@ func (s *Store) RenameAppIn(accountID int64, from, to string) error { |
| to, cliKindApp, cliKindService, accountID, from); err != nil { |
| return err |
| } |
| + var sqid string |
| + if err := tx.QueryRow(`SELECT sqid FROM accounts WHERE id = ?`, accountID).Scan(&sqid); err != nil { |
| + return err |
| + } |
| + if err := carryMailApp(context.Background(), tx, sqid, from, sqid, to, createdAt); err != nil { |
| + return err |
| + } |
| return tx.Commit() |
| } |
diff --git a/internal/mail/events.go b/internal/mail/events.go| index 959d5f4d9..4fb9b62a7 100644 |
| --- a/internal/mail/events.go |
| +++ b/internal/mail/events.go |
| @@ -8,14 +8,17 @@ import ( |
| "fmt" |
| "strings" |
| "time" |
| + "unicode" |
| + "unicode/utf8" |
| ) |
| const ( |
| - EventSend = "Send" |
| - EventBounce = "Bounce" |
| - EventComplaint = "Complaint" |
| - EventReject = "Reject" |
| - EventDelivery = "Delivery" |
| + EventSend = "Send" |
| + EventBounce = "Bounce" |
| + EventComplaint = "Complaint" |
| + EventReject = "Reject" |
| + EventDelivery = "Delivery" |
| + EventDeliveryDelay = "DeliveryDelay" |
| ) |
| // Event is the part of an SES notification this package acts on. |
| @@ -24,7 +27,84 @@ type Event struct { |
| Domain string |
| From string |
| MessageID string |
| - At time.Time |
| + // At is when what the event reports happened: the bounce, the delivery, |
| + // the complaint — falling back to the send, which is all a Send or a |
| + // Reject has. SentAt is always the send. |
| + At time.Time |
| + SentAt time.Time |
| + |
| + // Caller is the IAM identity SES says sent the message — the automatic |
| + // "ses:caller-identity" tag. CARLOS mints one per app (UserName), so |
| + // where the sending domain can only name every app that shares it, this |
| + // names the one app that actually sent. Empty when SES did not say; |
| + // AppOfCaller reads it back. |
| + Caller string |
| + // Destination is who the message was addressed to (mail.destination). |
| + Destination []string |
| + // Outcomes is what this event says about each recipient it concerns: |
| + // the bounced, complained-about, delivered or delayed addresses, each |
| + // with what the far end said. A Send or a Reject is about the whole |
| + // message and carries none. |
| + Outcomes []Outcome |
| + // Detail is the event-wide explanation, empty where the type has none. |
| + Detail EventDetail |
| + // Incomplete says why part of what only the log uses could not be read — |
| + // a field in a shape this code did not expect. The event is still good |
| + // for the counts; the caller logs this. |
| + Incomplete string |
| +} |
| + |
| +// Outcome is one recipient's part in an event. |
| +type Outcome struct { |
| + Recipient string |
| + // Status is the enhanced status code the receiving server gave, e.g. |
| + // "5.1.1". Empty on a delivery and a complaint. |
| + Status string |
| + // Diagnostic is the receiving server's own words: the bounce or delay |
| + // diagnostic, or the SMTP response it accepted a delivery with. |
| + Diagnostic string |
| +} |
| + |
| +// EventDetail is what an event says about the message as a whole. |
| +type EventDetail struct { |
| + BounceType string // Permanent, Transient, Undetermined |
| + BounceSubType string // General, NoEmail, Suppressed, MailboxFull, ... |
| + ComplaintType string // abuse, fraud, not-spam, ... when the provider said |
| + RejectReason string // e.g. "Bad content" |
| + DelayType string // e.g. MailboxFull, TransientCommunicationFailure |
| +} |
| + |
| +// The longest a field from an event is kept at. An address is at most 254 |
| +// octets by RFC 5321; a diagnostic is the far end's free text and has no |
| +// limit of its own, so it gets one here — enough for any real explanation, |
| +// not enough for a receiving server to fill a table with. |
| +const ( |
| + maxAddressLen = 254 |
| + maxDiagnosticLen = 1000 |
| + maxDetailLen = 64 |
| +) |
| + |
| +// cleanField drops control characters and bounds the length. Every string |
| +// that comes out of an event and is kept goes through it: the event is |
| +// signed by SNS, but the text in it was written by whichever mail server |
| +// answered, and it ends up in a table and on a terminal. |
| +func cleanField(s string, max int) string { |
| + s = strings.Map(func(r rune) rune { |
| + // C0, DEL and C1 (U+009B is a CSI on some terminals), and the |
| + // invisible formatting class — bidi overrides among it — which can |
| + // make a line on a terminal say something the bytes do not. |
| + if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) { |
| + return ' ' |
| + } |
| + return r |
| + }, strings.TrimSpace(s)) |
| + if len(s) > max { |
| + s = s[:max] |
| + for len(s) > 0 && !utf8.ValidString(s) { |
| + s = s[:len(s)-1] |
| + } |
| + } |
| + return s |
| } |
| // DomainOf extracts the lowercase domain from an address, tolerating a |
| @@ -67,6 +147,10 @@ func DomainOf(address string) string { |
| // is worse than no event, because it would be silently dropped from the |
| // counts that enforce a cap. |
| func ParseEvent(body []byte) (Event, error) { |
| + // Two reads of the same body, deliberately. This one is everything the |
| + // counts and the cap depend on, typed exactly as it was before the log |
| + // existed, and it is the only one allowed to refuse the event: a refused |
| + // event is a count silently dropped. |
| var raw struct { |
| EventType string `json:"eventType"` |
| Mail struct { |
| @@ -121,8 +205,175 @@ func ParseEvent(body []byte) (Event, error) { |
| } |
| if raw.Mail.Timestamp != "" { |
| if at, err := time.Parse(time.RFC3339, raw.Mail.Timestamp); err == nil { |
| + ev.SentAt = at |
| ev.At = at |
| } |
| } |
| + ev.MessageID = cleanField(ev.MessageID, maxDiagnosticLen) |
| + readLogFields(body, raw.EventType, &ev) |
| return ev, nil |
| } |
| + |
| +// readLogFields is the second read: what only the per-message log uses — |
| +// who the message went to, who sent it, and what the receiving server said. |
| +// A field here in a shape this code does not expect is not a reason to lose |
| +// the event, so nothing here refuses it: the part that does not read is left |
| +// empty, the reason is kept in ev.Incomplete for the caller to log, and the |
| +// count goes ahead. The envelope and the event's own block are read |
| +// separately, so a malformed bounce block does not also cost the sender. |
| +func readLogFields(body []byte, eventType string, ev *Event) { |
| + type recipient struct { |
| + EmailAddress string `json:"emailAddress"` |
| + Status string `json:"status"` |
| + DiagnosticCode string `json:"diagnosticCode"` |
| + } |
| + var problems []string |
| + var envelope struct { |
| + Mail struct { |
| + Destination []string `json:"destination"` |
| + Tags map[string][]string `json:"tags"` |
| + } `json:"mail"` |
| + } |
| + if err := json.Unmarshal(body, &envelope); err != nil { |
| + problems = append(problems, "mail: "+err.Error()) |
| + } |
| + if ids := envelope.Mail.Tags["ses:caller-identity"]; len(ids) > 0 { |
| + ev.Caller = cleanField(ids[0], maxDiagnosticLen) |
| + } |
| + for _, to := range envelope.Mail.Destination { |
| + if to = cleanField(to, maxAddressLen); to != "" { |
| + ev.Destination = append(ev.Destination, to) |
| + } |
| + } |
| + |
| + // Each type carries its own block; the others are simply empty. Read |
| + // only the one this event is, so a malformed message that carries two |
| + // cannot have one type's recipients reported under another's name. |
| + var raw struct { |
| + Bounce struct { |
| + Timestamp string `json:"timestamp"` |
| + BounceType string `json:"bounceType"` |
| + BounceSubType string `json:"bounceSubType"` |
| + BouncedRecipients []recipient `json:"bouncedRecipients"` |
| + } `json:"bounce"` |
| + Complaint struct { |
| + Timestamp string `json:"timestamp"` |
| + ComplaintFeedbackType string `json:"complaintFeedbackType"` |
| + ComplainedRecipients []recipient `json:"complainedRecipients"` |
| + } `json:"complaint"` |
| + Delivery struct { |
| + Timestamp string `json:"timestamp"` |
| + Recipients []string `json:"recipients"` |
| + SMTPResponse string `json:"smtpResponse"` |
| + } `json:"delivery"` |
| + Reject struct { |
| + Reason string `json:"reason"` |
| + } `json:"reject"` |
| + DeliveryDelay struct { |
| + Timestamp string `json:"timestamp"` |
| + DelayType string `json:"delayType"` |
| + DelayedRecipients []recipient `json:"delayedRecipients"` |
| + } `json:"deliveryDelay"` |
| + } |
| + block := map[string]string{ |
| + EventBounce: "bounce", EventComplaint: "complaint", EventDelivery: "delivery", |
| + EventReject: "reject", EventDeliveryDelay: "deliveryDelay", |
| + }[eventType] |
| + if block != "" { |
| + var blocks map[string]json.RawMessage |
| + if err := json.Unmarshal(body, &blocks); err == nil { |
| + // Wrap the one block back under its own key and read only that. |
| + if one, ok := blocks[block]; ok { |
| + if err := json.Unmarshal([]byte(`{"`+block+`":`+string(one)+`}`), &raw); err != nil { |
| + problems = append(problems, block+": "+err.Error()) |
| + } |
| + } |
| + } |
| + } |
| + if len(problems) > 0 { |
| + ev.Incomplete = cleanField(strings.Join(problems, "; "), maxDiagnosticLen) |
| + } |
| + |
| + // When the thing this event reports happened, where SES says: a bounce |
| + // or a delay can arrive a day after the send, and an event dated by the |
| + // send is both wrong on a screen and invisible to "since yesterday". |
| + eventTime := map[string]string{ |
| + EventBounce: raw.Bounce.Timestamp, EventComplaint: raw.Complaint.Timestamp, |
| + EventDelivery: raw.Delivery.Timestamp, EventDeliveryDelay: raw.DeliveryDelay.Timestamp, |
| + }[eventType] |
| + if eventTime != "" { |
| + if at, err := time.Parse(time.RFC3339, eventTime); err == nil { |
| + ev.At = at |
| + } |
| + } |
| + |
| + outcomes := func(rs []recipient) { |
| + for _, r := range rs { |
| + to := cleanField(r.EmailAddress, maxAddressLen) |
| + if to == "" { |
| + continue |
| + } |
| + ev.Outcomes = append(ev.Outcomes, Outcome{Recipient: to, |
| + Status: cleanField(r.Status, maxDetailLen), Diagnostic: cleanField(r.DiagnosticCode, maxDiagnosticLen)}) |
| + } |
| + } |
| + switch eventType { |
| + case EventBounce: |
| + ev.Detail.BounceType = cleanField(raw.Bounce.BounceType, maxDetailLen) |
| + ev.Detail.BounceSubType = cleanField(raw.Bounce.BounceSubType, maxDetailLen) |
| + outcomes(raw.Bounce.BouncedRecipients) |
| + case EventComplaint: |
| + ev.Detail.ComplaintType = cleanField(raw.Complaint.ComplaintFeedbackType, maxDetailLen) |
| + outcomes(raw.Complaint.ComplainedRecipients) |
| + case EventDelivery: |
| + said := cleanField(raw.Delivery.SMTPResponse, maxDiagnosticLen) |
| + for _, to := range raw.Delivery.Recipients { |
| + if to = cleanField(to, maxAddressLen); to != "" { |
| + ev.Outcomes = append(ev.Outcomes, Outcome{Recipient: to, Diagnostic: said}) |
| + } |
| + } |
| + case EventReject: |
| + ev.Detail.RejectReason = cleanField(raw.Reject.Reason, maxDiagnosticLen) |
| + case EventDeliveryDelay: |
| + ev.Detail.DelayType = cleanField(raw.DeliveryDelay.DelayType, maxDetailLen) |
| + outcomes(raw.DeliveryDelay.DelayedRecipients) |
| + } |
| +} |
| + |
| +// Recipients is who this event is about, one entry per address: the |
| +// recipients it reports an outcome for, or — for an event about the whole |
| +// message, a Send or a Reject — everyone it was addressed to. A per-recipient |
| +// event (a bounce, a delivery, a complaint, a delay) that names nobody is NOT |
| +// widened to every addressee: that would report a bounce for addresses that |
| +// did not bounce. It, like any event that names nobody at all, yields a |
| +// single empty entry, so it is still recorded once rather than not at all. |
| +func (ev Event) Recipients() []Outcome { |
| + if len(ev.Outcomes) > 0 { |
| + return ev.Outcomes |
| + } |
| + wholeMessage := ev.Type == EventSend || ev.Type == EventReject |
| + if len(ev.Destination) == 0 || !wholeMessage { |
| + return []Outcome{{}} |
| + } |
| + out := make([]Outcome, 0, len(ev.Destination)) |
| + for _, to := range ev.Destination { |
| + out = append(out, Outcome{Recipient: to}) |
| + } |
| + return out |
| +} |
| + |
| +// Summary is Detail in a few words, for a table: "Permanent/NoEmail", |
| +// "abuse", "Bad content". Empty when the event has nothing to add. |
| +func (d EventDetail) Summary() string { |
| + switch { |
| + case d.BounceType != "" && d.BounceSubType != "": |
| + return d.BounceType + "/" + d.BounceSubType |
| + case d.BounceType != "": |
| + return d.BounceType |
| + case d.ComplaintType != "": |
| + return d.ComplaintType |
| + case d.RejectReason != "": |
| + return d.RejectReason |
| + } |
| + return d.DelayType |
| +} |
diff --git a/internal/mail/events_test.go b/internal/mail/events_test.go| index f23326d3c..34143db3e 100644 |
| --- a/internal/mail/events_test.go |
| +++ b/internal/mail/events_test.go |
| @@ -1,6 +1,11 @@ |
| package mail |
| -import "testing" |
| +import ( |
| + "encoding/json" |
| + "strings" |
| + "testing" |
| + "time" |
| +) |
| // SES's event JSON nests the From address under mail.source and the type |
| // under eventType. Attribution is by SENDING DOMAIN, because that is the |
| @@ -161,3 +166,248 @@ func TestParseEventIgnoresMalformedTimestamp(t *testing.T) { |
| t.Fatalf("At = %v, want zero (timestamp ignored)", ev.At) |
| } |
| } |
| + |
| +// What SES publishes for a bounce, in the shape its event destinations |
| +// send: the message's own envelope under mail, the automatic tags naming |
| +// the sending identity, and a bounce block that says, per recipient, what |
| +// the receiving server answered. |
| +const bounceEvent = `{ |
| + "eventType": "Bounce", |
| + "bounce": { |
| + "bounceType": "Permanent", |
| + "bounceSubType": "NoEmail", |
| + "bouncedRecipients": [ |
| + {"emailAddress": "ann@example.com", "action": "failed", "status": "5.1.1", |
| + "diagnosticCode": "smtp; 550 5.1.1 <ann@example.com>: Recipient address rejected"} |
| + ], |
| + "timestamp": "2026-09-15T08:00:02.000Z", |
| + "feedbackId": "0102019-feedback" |
| + }, |
| + "mail": { |
| + "timestamp": "2026-09-15T08:00:00.000Z", |
| + "source": "bookings@lgc.li", |
| + "messageId": "0102019abc-7c8d-000000", |
| + "destination": ["ann@example.com"], |
| + "tags": { |
| + "ses:caller-identity": ["carlos-mail-beb-jb-book"], |
| + "ses:from-domain": ["lgc.li"] |
| + } |
| + } |
| +}` |
| + |
| +func TestParseEventReadsWhoBouncedAndWhy(t *testing.T) { |
| + ev, err := ParseEvent([]byte(bounceEvent)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if ev.MessageID != "0102019abc-7c8d-000000" || ev.Caller != "carlos-mail-beb-jb-book" { |
| + t.Errorf("message %q, caller %q", ev.MessageID, ev.Caller) |
| + } |
| + if len(ev.Outcomes) != 1 { |
| + t.Fatalf("outcomes = %+v, want one", ev.Outcomes) |
| + } |
| + o := ev.Outcomes[0] |
| + if o.Recipient != "ann@example.com" || o.Status != "5.1.1" || !strings.Contains(o.Diagnostic, "Recipient address rejected") { |
| + t.Errorf("outcome = %+v", o) |
| + } |
| + if got := ev.Detail.Summary(); got != "Permanent/NoEmail" { |
| + t.Errorf("summary = %q", got) |
| + } |
| +} |
| + |
| +func TestParseEventReadsEachTypesOwnRecipients(t *testing.T) { |
| + for _, c := range []struct { |
| + name, body, recipient, diagnostic, summary string |
| + }{ |
| + {"delivery", `{"eventType":"Delivery","mail":{"source":"a@lgc.li","messageId":"m1","destination":["bob@example.com"]}, |
| + "delivery":{"recipients":["bob@example.com"],"smtpResponse":"250 2.0.0 OK 1726387202"}}`, |
| + "bob@example.com", "250 2.0.0 OK 1726387202", ""}, |
| + {"complaint", `{"eventType":"Complaint","mail":{"source":"a@lgc.li","messageId":"m2"}, |
| + "complaint":{"complaintFeedbackType":"abuse","complainedRecipients":[{"emailAddress":"cat@example.com"}]}}`, |
| + "cat@example.com", "", "abuse"}, |
| + {"delay", `{"eventType":"DeliveryDelay","mail":{"source":"a@lgc.li","messageId":"m3"}, |
| + "deliveryDelay":{"delayType":"MailboxFull","delayedRecipients":[{"emailAddress":"dan@example.com","status":"4.2.2","diagnosticCode":"452 mailbox full"}]}}`, |
| + "dan@example.com", "452 mailbox full", "MailboxFull"}, |
| + } { |
| + t.Run(c.name, func(t *testing.T) { |
| + ev, err := ParseEvent([]byte(c.body)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + rs := ev.Recipients() |
| + if len(rs) != 1 || rs[0].Recipient != c.recipient || rs[0].Diagnostic != c.diagnostic { |
| + t.Errorf("recipients = %+v", rs) |
| + } |
| + if got := ev.Detail.Summary(); got != c.summary { |
| + t.Errorf("summary = %q, want %q", got, c.summary) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// A Send or a Reject is about the whole message, so it concerns everyone |
| +// the message was addressed to. An event that names nobody is still one |
| +// event, not none. |
| +func TestAnEventAboutTheWholeMessageConcernsEveryAddressee(t *testing.T) { |
| + ev, err := ParseEvent([]byte(`{"eventType":"Reject","mail":{"source":"a@lgc.li","messageId":"m4", |
| + "destination":["one@example.com","two@example.com"]},"reject":{"reason":"Bad content"}}`)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if rs := ev.Recipients(); len(rs) != 2 || rs[1].Recipient != "two@example.com" { |
| + t.Errorf("recipients = %+v", rs) |
| + } |
| + if ev.Detail.Summary() != "Bad content" { |
| + t.Errorf("summary = %q", ev.Detail.Summary()) |
| + } |
| + bare, _ := ParseEvent([]byte(`{"eventType":"Send","mail":{"source":"a@lgc.li"}}`)) |
| + if rs := bare.Recipients(); len(rs) != 1 || rs[0].Recipient != "" { |
| + t.Errorf("an event naming nobody should still be one entry: %+v", rs) |
| + } |
| +} |
| + |
| +// A type's recipients are read from its own block only. A malformed |
| +// message carrying a delivery block under a bounce must not report the |
| +// delivered address as bounced. |
| +func TestParseEventReadsOnlyTheBlockItsTypeNames(t *testing.T) { |
| + ev, err := ParseEvent([]byte(`{"eventType":"Bounce","mail":{"source":"a@lgc.li"}, |
| + "delivery":{"recipients":["bob@example.com"]}}`)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(ev.Outcomes) != 0 { |
| + t.Errorf("a bounce reported the delivery block's recipients: %+v", ev.Outcomes) |
| + } |
| +} |
| + |
| +// The text in an event was written by whichever server answered. It is |
| +// kept, so it is kept clean: no control characters, and bounded. |
| +func TestEventTextIsCleanedBeforeItIsKept(t *testing.T) { |
| + long := strings.Repeat("x", 5000) |
| + ev, err := ParseEvent([]byte(`{"eventType":"Bounce","mail":{"source":"a@lgc.li"},"bounce":{"bounceType":"Permanent", |
| + "bouncedRecipients":[{"emailAddress":"ann@example.com\r\nX-Injected: yes","diagnosticCode":"` + long + `"}]}}`)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + o := ev.Outcomes[0] |
| + if strings.ContainsAny(o.Recipient, "\r\n") { |
| + t.Errorf("control characters survived: %q", o.Recipient) |
| + } |
| + if len(o.Diagnostic) > maxDiagnosticLen { |
| + t.Errorf("diagnostic kept at %d bytes", len(o.Diagnostic)) |
| + } |
| +} |
| + |
| +func TestAppOfCallerReadsBackTheDeliveredIdentity(t *testing.T) { |
| + for _, c := range []struct { |
| + caller, acct, app string |
| + ok bool |
| + }{ |
| + {UserName("beb", "jb-book"), "beb", "jb-book", true}, |
| + {StandaloneUserName("beb", "jb-book", "laptop"), "beb", "jb-book", true}, |
| + {"arn:aws:iam::123456789012:user/carlos/mail/carlos-mail-beb-jb-book", "beb", "jb-book", true}, |
| + {"arn:aws:sts::123456789012:assumed-role/x/carlos-mail-beb-jb-book", "", "", false}, // a role session names itself |
| + {"arn:aws:iam::123456789012:role/carlos-mail-beb-jb-book", "", "", false}, // not a user |
| + {"carlos/mail/carlos-mail-beb-jb-book", "", "", false}, // a path with no ARN |
| + {"carlos-mail-beb-jb-book+s+", "", "", false}, // a standalone with no name |
| + {"carlos-mail-beb", "", "", false}, // no app |
| + {"carlos-mail-BEB-jb-book", "", "", false}, // not an account sqid |
| + {"someone-else-entirely", "", "", false}, // not ours |
| + {"", "", "", false}, |
| + } { |
| + acct, app, ok := AppOfCaller(c.caller) |
| + if acct != c.acct || app != c.app || ok != c.ok { |
| + t.Errorf("AppOfCaller(%q) = %q, %q, %v; want %q, %q, %v", c.caller, acct, app, ok, c.acct, c.app, c.ok) |
| + } |
| + } |
| +} |
| + |
| +// A terminal can be made to show something the bytes do not say: an 8-bit |
| +// CSI, or a right-to-left override that reverses the rest of the line. The |
| +// far end writes the diagnostic, so none of that is kept. |
| +func TestEventTextLosesInvisibleAndEightBitControls(t *testing.T) { |
| + sneaky := []rune{0x9b, 0x202e, 0x202c, 0x200b} |
| + diag := "550 " + string(sneaky[0]) + "31m red " + string(sneaky[1]) + "evil" + string(sneaky[2]) + string(sneaky[3]) + " ok" |
| + quoted, _ := json.Marshal(diag) |
| + ev, err := ParseEvent([]byte(`{"eventType":"Bounce","mail":{"source":"a@lgc.li"},"bounce":{"bouncedRecipients":[ |
| + {"emailAddress":"ann@example.com","diagnosticCode":` + string(quoted) + `}]}}`)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, r := range ev.Outcomes[0].Diagnostic { |
| + for _, bad := range sneaky { |
| + if r == bad { |
| + t.Errorf("%U survived in %q", r, ev.Outcomes[0].Diagnostic) |
| + } |
| + } |
| + } |
| +} |
| + |
| +// A bounce can come a day after the send. The event is dated by the bounce, |
| +// and the send is kept beside it. |
| +func TestAnEventIsDatedByWhatItReports(t *testing.T) { |
| + ev, err := ParseEvent([]byte(`{"eventType":"Bounce","mail":{"source":"a@lgc.li","timestamp":"2026-09-14T08:00:00Z"}, |
| + "bounce":{"timestamp":"2026-09-15T09:30:00Z","bouncedRecipients":[{"emailAddress":"ann@example.com"}]}}`)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if got := ev.At.UTC().Format(time.RFC3339); got != "2026-09-15T09:30:00Z" { |
| + t.Errorf("At = %s, want the bounce's own time", got) |
| + } |
| + if got := ev.SentAt.UTC().Format(time.RFC3339); got != "2026-09-14T08:00:00Z" { |
| + t.Errorf("SentAt = %s, want the send", got) |
| + } |
| + send, _ := ParseEvent([]byte(`{"eventType":"Send","mail":{"source":"a@lgc.li","timestamp":"2026-09-14T08:00:00Z"}}`)) |
| + if !send.At.Equal(send.SentAt) || send.At.IsZero() { |
| + t.Errorf("a Send is dated by the send: %v / %v", send.At, send.SentAt) |
| + } |
| +} |
| + |
| +// What only the log reads must never cost the count. A notification whose |
| +// tags, destination or type block arrive in a shape this code does not |
| +// expect still parses for everything the counts and the cap depend on, and |
| +// says what it could not read. |
| +func TestAMisshapedLogFieldDoesNotCostTheEvent(t *testing.T) { |
| + for name, body := range map[string]string{ |
| + "tags as a string": `{"eventType":"Send","mail":{"source":"a@lgc.li","timestamp":"2026-09-15T08:00:00.000Z","tags":{"ses:caller-identity":"x"}}}`, |
| + "destination as a string": `{"eventType":"Send","mail":{"source":"a@lgc.li","timestamp":"2026-09-15T08:00:00.000Z","destination":"ann@example.com"}}`, |
| + "recipients as objects": `{"eventType":"Delivery","mail":{"source":"a@lgc.li","timestamp":"2026-09-15T08:00:00.000Z"},"delivery":{"recipients":[{"a":1}]}}`, |
| + "reason as an object": `{"eventType":"Reject","mail":{"source":"a@lgc.li","timestamp":"2026-09-15T08:00:00.000Z"},"reject":{"reason":{"x":1}}}`, |
| + "bounce type as a number": `{"eventType":"Bounce","mail":{"source":"a@lgc.li","timestamp":"2026-09-15T08:00:00.000Z","tags":{"ses:caller-identity":["carlos-mail-beb-jb-book"]}},"bounce":{"bounceType":7}}`, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + ev, err := ParseEvent([]byte(body)) |
| + if err != nil { |
| + t.Fatalf("refused — a count would be dropped for a field only the log reads: %v", err) |
| + } |
| + if ev.Type == "" || ev.Domain != "lgc.li" || ev.SentAt.IsZero() { |
| + t.Errorf("the fields the counts need did not survive: %+v", ev) |
| + } |
| + if ev.Incomplete == "" { |
| + t.Error("what could not be read should be said, for the log") |
| + } |
| + }) |
| + } |
| + // A malformed type block does not also cost the envelope beside it. |
| + ev, _ := ParseEvent([]byte(`{"eventType":"Bounce","mail":{"source":"a@lgc.li","tags":{"ses:caller-identity":["carlos-mail-beb-jb-book"]}},"bounce":{"bounceType":7}}`)) |
| + if ev.Caller != "carlos-mail-beb-jb-book" { |
| + t.Errorf("a malformed bounce block cost the sender: %q", ev.Caller) |
| + } |
| + good, _ := ParseEvent([]byte(bounceEvent)) |
| + if good.Incomplete != "" { |
| + t.Errorf("a well-formed event claims to be incomplete: %q", good.Incomplete) |
| + } |
| +} |
| + |
| +// A bounce is about the recipients it names. One that names nobody is not a |
| +// bounce for everybody the message was addressed to. |
| +func TestAPerRecipientEventThatNamesNobodyIsNotWidened(t *testing.T) { |
| + ev, err := ParseEvent([]byte(`{"eventType":"Bounce","mail":{"source":"a@lgc.li","messageId":"m", |
| + "destination":["one@example.com","two@example.com","three@example.com"]},"bounce":{"bounceType":"Permanent","bouncedRecipients":[]}}`)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if rs := ev.Recipients(); len(rs) != 1 || rs[0].Recipient != "" { |
| + t.Errorf("a bounce naming nobody was widened to %+v", rs) |
| + } |
| +} |
diff --git a/internal/mail/policy.go b/internal/mail/policy.go| index 82df3eacc..7654d3923 100644 |
| --- a/internal/mail/policy.go |
| +++ b/internal/mail/policy.go |
| @@ -9,6 +9,8 @@ import ( |
| "encoding/json" |
| "fmt" |
| "strings" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| ) |
| // PolicyName is the single inline policy per user. One name, one document — |
| @@ -24,6 +26,50 @@ const UserPath = "/carlos-mail/" |
| // UserName is the principal that owns an app's DELIVERED credentials. |
| func UserName(acct, app string) string { return "carlos-mail-" + acct + "-" + app } |
| +// AppOfCaller reads the account and app back out of the IAM identity SES |
| +// reports as a message's sender (Event.Caller) — the inverse of UserName, |
| +// and of StandaloneUserName, whose credential sends for the same app. It is |
| +// unambiguous because an account sqid is lowercase letters only |
| +// (release.ValidAccount), so the first hyphen after the prefix is where the |
| +// account ends, and everything after it is the app, hyphens and all. |
| +// |
| +// SES is documented to report the identity's name. An IAM USER ARN is |
| +// accepted too, by its last path segment, so a change in which of the two |
| +// SES sends does not quietly stop naming the sender — but only a user ARN: |
| +// an assumed-role session is named by whoever assumes the role, and a |
| +// session named carlos-mail-<acct>-<app> is not that app. Anything that is |
| +// not one of CARLOS's own identities answers false. |
| +func AppOfCaller(caller string) (acct, app string, ok bool) { |
| + name := caller |
| + if strings.HasPrefix(name, "arn:") { |
| + _, resource, found := strings.Cut(name, ":user/") |
| + if !found || !strings.HasPrefix(name, "arn:aws") || strings.Count(name[:strings.Index(name, ":user/")], ":") != 4 { |
| + return "", "", false |
| + } |
| + name = resource |
| + if i := strings.LastIndex(name, "/"); i >= 0 { |
| + name = name[i+1:] |
| + } |
| + } else if strings.Contains(name, "/") { |
| + return "", "", false |
| + } |
| + rest, found := strings.CutPrefix(name, "carlos-mail-") |
| + if !found { |
| + return "", "", false |
| + } |
| + if i := strings.Index(rest, "+s+"); i >= 0 { |
| + if rest[i+3:] == "" { |
| + return "", "", false // a standalone credential always has a name |
| + } |
| + rest = rest[:i] |
| + } |
| + acct, app, found = strings.Cut(rest, "-") |
| + if !found || !release.ValidAccount(acct) || !release.ValidApp(app) { |
| + return "", "", false |
| + } |
| + return acct, app, true |
| +} |
| + |
| // TenantName is the SES tenant CARLOS creates for a CARLOS account. |
| // |
| // It is prefixed, never the bare sqid, because the SES tenant namespace is |
diff --git a/mail_cli.go b/mail_cli.go| index 1871adf4d..1a96f201f 100644 |
| --- a/mail_cli.go |
| +++ b/mail_cli.go |
| @@ -48,8 +48,10 @@ import ( |
| "net" |
| "net/http" |
| "net/smtp" |
| + "net/url" |
| "os" |
| "sort" |
| + "strconv" |
| "strings" |
| "text/tabwriter" |
| "time" |
| @@ -76,6 +78,12 @@ const emailUsage = `usage: carlos email <enable|domains|status|test|credentials| |
| status --app <a> domains and their per-region verification state, |
| today's counts against the cap, and the pause |
| state. |
| + events --app <a> [--since <dur|time>] [--message <id>] [--to <addr>] [-n <count>] [--json] |
| + what happened to each message the app sent — |
| + delivered, bounced, complained about, delayed — |
| + newest first, with what the receiving server |
| + said. Kept for 30 days. --message takes the id SES |
| + gave the message when it was sent. |
| test --app <a> --to <addr> |
| send a REAL message and report the result — the |
| verify half of the no-infra rule: this is the |
| @@ -126,6 +134,8 @@ func runEmail(args []string) { |
| runEmailDomains(args[1:]) |
| case "status": |
| runEmailStatus(args[1:]) |
| + case "events": |
| + runEmailEvents(args[1:]) |
| case "test": |
| runEmailTest(args[1:]) |
| case "credentials": |
| @@ -804,6 +814,173 @@ func runEmailStatus(args []string) { |
| printEmailStatus(stdout, res) |
| } |
| +// apiEmailEvent is one row of `carlos email events`, as the console sends it. |
| +type apiEmailEvent struct { |
| + MessageID string `json:"message_id"` |
| + Type string `json:"type"` |
| + Recipient string `json:"recipient"` |
| + Status string `json:"status,omitempty"` |
| + Diagnostic string `json:"diagnostic,omitempty"` |
| + Detail string `json:"detail,omitempty"` |
| + BySender bool `json:"by_sender"` |
| + At time.Time `json:"at"` |
| +} |
| + |
| +type apiEmailEventsReply struct { |
| + Events []apiEmailEvent `json:"events"` |
| + RetentionDays int `json:"retention_days"` |
| +} |
| + |
| +func (s *apiSession) emailEvents(ctx context.Context, a apiAcct, app string, q url.Values) (apiEmailEventsReply, error) { |
| + tail := "/email/events" |
| + if len(q) > 0 { |
| + tail += "?" + q.Encode() |
| + } |
| + status, body, err := s.getJSON(ctx, apiPath(a, app, tail)) |
| + if err != nil { |
| + return apiEmailEventsReply{}, err |
| + } |
| + if status != http.StatusOK { |
| + return apiEmailEventsReply{}, mailAPIError(status, body) |
| + } |
| + var out apiEmailEventsReply |
| + if err := json.Unmarshal(body, &out); err != nil { |
| + return apiEmailEventsReply{}, fmt.Errorf("email events: unreadable response: %w", err) |
| + } |
| + return out, nil |
| +} |
| + |
| +// sinceParam turns --since into the RFC 3339 time the console reads: a |
| +// duration counts back from now, anything else must already be a time. |
| +// Empty stays empty — the whole retention window. |
| +func sinceParam(raw string, now time.Time) (string, error) { |
| + raw = strings.TrimSpace(raw) |
| + if raw == "" { |
| + return "", nil |
| + } |
| + if d, err := time.ParseDuration(raw); err == nil { |
| + if d < 0 { |
| + d = -d |
| + } |
| + return now.Add(-d).UTC().Format(time.RFC3339), nil |
| + } |
| + if t, err := time.Parse(time.RFC3339, raw); err == nil { |
| + return t.UTC().Format(time.RFC3339), nil |
| + } |
| + return "", fmt.Errorf("--since %q is neither a duration (24h) nor a time (2026-09-15T08:00:00Z)", raw) |
| +} |
| + |
| +func runEmailEvents(args []string) { |
| + fs := flag.NewFlagSet("email events", flag.ExitOnError) |
| + app := appFlag(fs, "app whose email events to show") |
| + acctRaw, acctIDRaw, _ := accountFlags(fs) |
| + since := fs.String("since", "", "only events since: a duration back from now (24h), or an RFC 3339 time (default: all 30 days kept)") |
| + message := fs.String("message", "", "only this message: the id SES gave it when it was sent") |
| + to := fs.String("to", "", "only events about this recipient address") |
| + n := fs.Int("n", 0, "at most this many events (default 100, max 1000)") |
| + asJSON := fs.Bool("json", false, "print the events as JSON, one array, for a script") |
| + consoleFlag := fs.String("console", "", "console origin (API mode only; default: the only logged-in one)") |
| + fs.Usage = flagUsage(fs, "") |
| + fs.Parse(args) |
| + name := strings.TrimSpace(*app) |
| + usage := "usage: carlos email events --app <a> [--since <dur|time>] [--message <id>] [--to <addr>] [-n <count>] [--json] [--account <sqid-or-name>] [--console <origin>]" |
| + if name != "" { |
| + if err := validAppName(name); err != nil { |
| + log.Fatalf("email events: %v", err) |
| + } |
| + } |
| + q := url.Values{} |
| + sinceAt, err := sinceParam(*since, time.Now()) |
| + if err != nil { |
| + log.Fatalf("email events: %v", err) |
| + } |
| + if sinceAt != "" { |
| + q.Set("since", sinceAt) |
| + } |
| + if v := strings.TrimSpace(*message); v != "" { |
| + q.Set("message", v) |
| + } |
| + if v := strings.TrimSpace(*to); v != "" { |
| + q.Set("to", v) |
| + } |
| + if *n > 0 { |
| + q.Set("limit", strconv.Itoa(*n)) |
| + } |
| + |
| + sess := requireMailAPISession(*consoleFlag) |
| + if !*asJSON { |
| + printUsing(stdout, sess) |
| + } |
| + acct := resolvedAccount(sess, acctRaw, acctIDRaw) |
| + name = resolvedApp(sess, acct, name, usage) |
| + res, err := sess.emailEvents(context.Background(), acct, name, q) |
| + if err != nil { |
| + log.Fatalf("email events: %v", err) |
| + } |
| + if *asJSON { |
| + if res.Events == nil { |
| + res.Events = []apiEmailEvent{} |
| + } |
| + enc := json.NewEncoder(stdout) |
| + enc.SetIndent("", " ") |
| + enc.Encode(res.Events) |
| + return |
| + } |
| + printEmailEvents(stdout, name, res) |
| +} |
| + |
| +// printEmailEvents is one line per recipient per event, newest first: when, |
| +// what happened, to whom, and the message it happened to — then, indented |
| +// beneath, what the receiving server actually said, which is the part that |
| +// tells a member what to do about a bounce. |
| +func printEmailEvents(w io.Writer, app string, res apiEmailEventsReply) { |
| + days := res.RetentionDays |
| + if days == 0 { |
| + days = 30 |
| + } |
| + if len(res.Events) == 0 { |
| + mutedf(w, "no email events for %s in the last %d days that match — events arrive a few seconds after SES hears back from the receiving server", app, days) |
| + return |
| + } |
| + width := 0 |
| + for _, e := range res.Events { |
| + if n := len(eventWord(e)); n > width { |
| + width = n |
| + } |
| + } |
| + for _, e := range res.Events { |
| + to := e.Recipient |
| + if to == "" { |
| + to = "(no recipient named)" |
| + } |
| + fmt.Fprintf(w, "%s %-*s %s %s\n", e.At.Local().Format("2006-01-02 15:04:05"), width, eventWord(e), to, e.MessageID) |
| + said := e.Diagnostic |
| + if e.Status != "" && !strings.Contains(said, e.Status) { |
| + said = strings.TrimSpace(e.Status + " " + said) |
| + } |
| + if said != "" { |
| + mutedf(w, " %s", said) |
| + } |
| + } |
| + mutedf(w, "%d event(s), newest first — kept for %d days", len(res.Events), days) |
| +} |
| + |
| +// eventWord is the event type as a member reads it, with the kind of bounce |
| +// or complaint where SES said: "bounce (Permanent/NoEmail)". |
| +func eventWord(e apiEmailEvent) string { |
| + word := map[string]string{ |
| + "Send": "sent", "Delivery": "delivered", "Bounce": "bounced", "Complaint": "complaint", |
| + "Reject": "rejected", "DeliveryDelay": "delayed", |
| + }[e.Type] |
| + if word == "" { |
| + word = strings.ToLower(e.Type) |
| + } |
| + if e.Detail != "" { |
| + word += " (" + e.Detail + ")" |
| + } |
| + return word |
| +} |
| + |
| // printEmailStatus is the one view a member with no box access and no AWS |
| // console has of every domain's per-region verification state, today's |
| // counts against the cap, and the pause state. |
diff --git a/mail_cli_test.go b/mail_cli_test.go| index 39fe68999..8bb049461 100644 |
| --- a/mail_cli_test.go |
| +++ b/mail_cli_test.go |
| @@ -1108,3 +1108,57 @@ func TestPrintEmailStatusDoesNotWarnOnAnUnverifiedDomain(t *testing.T) { |
| t.Fatalf("status output warns about events on a still-verifying domain:\n%s", out) |
| } |
| } |
| + |
| +// `carlos email events` puts what happened first and what the receiving |
| +// server said beneath it — the second line is what tells a member whether |
| +// a bounce was a typo in the address or a full mailbox. |
| +func TestPrintEmailEventsSaysWhatHappenedAndWhy(t *testing.T) { |
| + var buf bytes.Buffer |
| + at := time.Date(2026, 9, 15, 8, 0, 2, 0, time.UTC) |
| + printEmailEvents(&buf, "jb-book", apiEmailEventsReply{RetentionDays: 30, Events: []apiEmailEvent{ |
| + {MessageID: "0102-ann", Type: "Bounce", Recipient: "ann@example.com", Status: "5.1.1", |
| + Diagnostic: "smtp; 550 5.1.1 Recipient address rejected", Detail: "Permanent/NoEmail", At: at}, |
| + {MessageID: "0102-bob", Type: "Delivery", Recipient: "bob@example.com", Diagnostic: "250 2.0.0 OK", At: at.Add(-time.Minute)}, |
| + }}) |
| + out := buf.String() |
| + for _, want := range []string{ |
| + "bounced (Permanent/NoEmail)", "ann@example.com", "0102-ann", "Recipient address rejected", |
| + "delivered", "bob@example.com", "250 2.0.0 OK", |
| + "2 event(s)", "30 days", |
| + } { |
| + if !strings.Contains(out, want) { |
| + t.Errorf("output is missing %q:\n%s", want, out) |
| + } |
| + } |
| + if strings.Index(out, "0102-ann") > strings.Index(out, "0102-bob") { |
| + t.Errorf("events should print in the order the console sent them, newest first:\n%s", out) |
| + } |
| + // The status code is already in the diagnostic here; it is not said twice. |
| + if strings.Count(out, "5.1.1") != 1 { |
| + t.Errorf("the status code was repeated:\n%s", out) |
| + } |
| + |
| + buf.Reset() |
| + printEmailEvents(&buf, "jb-book", apiEmailEventsReply{RetentionDays: 30}) |
| + if !strings.Contains(buf.String(), "no email events for jb-book") { |
| + t.Errorf("an empty log should say so, and why it might be empty:\n%s", buf.String()) |
| + } |
| +} |
| + |
| +func TestSinceParamTakesADurationOrATime(t *testing.T) { |
| + now := time.Date(2026, 9, 15, 12, 0, 0, 0, time.UTC) |
| + for _, c := range []struct{ in, want string }{ |
| + {"", ""}, |
| + {"24h", "2026-09-14T12:00:00Z"}, |
| + {"90m", "2026-09-15T10:30:00Z"}, |
| + {"2026-09-10T08:00:00+01:00", "2026-09-10T07:00:00Z"}, |
| + } { |
| + got, err := sinceParam(c.in, now) |
| + if err != nil || got != c.want { |
| + t.Errorf("sinceParam(%q) = %q, %v; want %q", c.in, got, err, c.want) |
| + } |
| + } |
| + if _, err := sinceParam("yesterday", now); err == nil { |
| + t.Error("a word that is neither should be refused, not sent to the console") |
| + } |
| +} |