| 1 | // SES event parsing (spec §9). Events are the abuse signal AND the only |
| 2 | // source of a per-tenant send count — SES has no per-identity rate limit to |
| 3 | // read, so a cap can only be enforced by counting what already went out. |
| 4 | package mail |
| 5 | |
| 6 | import ( |
| 7 | "encoding/json" |
| 8 | "fmt" |
| 9 | "strings" |
| 10 | "time" |
| 11 | "unicode" |
| 12 | "unicode/utf8" |
| 13 | ) |
| 14 | |
| 15 | const ( |
| 16 | EventSend = "Send" |
| 17 | EventBounce = "Bounce" |
| 18 | EventComplaint = "Complaint" |
| 19 | EventReject = "Reject" |
| 20 | EventDelivery = "Delivery" |
| 21 | EventDeliveryDelay = "DeliveryDelay" |
| 22 | ) |
| 23 | |
| 24 | // Event is the part of an SES notification this package acts on. |
| 25 | type Event struct { |
| 26 | Type string |
| 27 | Domain string |
| 28 | From string |
| 29 | MessageID string |
| 30 | // At is when what the event reports happened: the bounce, the delivery, |
| 31 | // the complaint — falling back to the send, which is all a Send or a |
| 32 | // Reject has. SentAt is always the send. |
| 33 | At time.Time |
| 34 | SentAt time.Time |
| 35 | |
| 36 | // Caller is the IAM identity SES says sent the message — the automatic |
| 37 | // "ses:caller-identity" tag. CARLOS mints one per app (UserName), so |
| 38 | // where the sending domain can only name every app that shares it, this |
| 39 | // names the one app that actually sent. Empty when SES did not say; |
| 40 | // AppOfCaller reads it back. |
| 41 | Caller string |
| 42 | // Destination is who the message was addressed to (mail.destination). |
| 43 | Destination []string |
| 44 | // Outcomes is what this event says about each recipient it concerns: |
| 45 | // the bounced, complained-about, delivered or delayed addresses, each |
| 46 | // with what the far end said. A Send or a Reject is about the whole |
| 47 | // message and carries none. |
| 48 | Outcomes []Outcome |
| 49 | // Detail is the event-wide explanation, empty where the type has none. |
| 50 | Detail EventDetail |
| 51 | // Incomplete says why part of what only the log uses could not be read — |
| 52 | // a field in a shape this code did not expect. The event is still good |
| 53 | // for the counts; the caller logs this. |
| 54 | Incomplete string |
| 55 | } |
| 56 | |
| 57 | // Outcome is one recipient's part in an event. |
| 58 | type Outcome struct { |
| 59 | Recipient string |
| 60 | // Status is the enhanced status code the receiving server gave, e.g. |
| 61 | // "5.1.1". Empty on a delivery and a complaint. |
| 62 | Status string |
| 63 | // Diagnostic is the receiving server's own words: the bounce or delay |
| 64 | // diagnostic, or the SMTP response it accepted a delivery with. |
| 65 | Diagnostic string |
| 66 | } |
| 67 | |
| 68 | // EventDetail is what an event says about the message as a whole. |
| 69 | type EventDetail struct { |
| 70 | BounceType string // Permanent, Transient, Undetermined |
| 71 | BounceSubType string // General, NoEmail, Suppressed, MailboxFull, ... |
| 72 | ComplaintType string // abuse, fraud, not-spam, ... when the provider said |
| 73 | RejectReason string // e.g. "Bad content" |
| 74 | DelayType string // e.g. MailboxFull, TransientCommunicationFailure |
| 75 | } |
| 76 | |
| 77 | // The longest a field from an event is kept at. An address is at most 254 |
| 78 | // octets by RFC 5321; a diagnostic is the far end's free text and has no |
| 79 | // limit of its own, so it gets one here — enough for any real explanation, |
| 80 | // not enough for a receiving server to fill a table with. |
| 81 | const ( |
| 82 | maxAddressLen = 254 |
| 83 | maxDiagnosticLen = 1000 |
| 84 | maxDetailLen = 64 |
| 85 | ) |
| 86 | |
| 87 | // cleanField drops control characters and bounds the length. Every string |
| 88 | // that comes out of an event and is kept goes through it: the event is |
| 89 | // signed by SNS, but the text in it was written by whichever mail server |
| 90 | // answered, and it ends up in a table and on a terminal. |
| 91 | func cleanField(s string, max int) string { |
| 92 | s = strings.Map(func(r rune) rune { |
| 93 | // C0, DEL and C1 (U+009B is a CSI on some terminals), and the |
| 94 | // invisible formatting class — bidi overrides among it — which can |
| 95 | // make a line on a terminal say something the bytes do not. |
| 96 | if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) { |
| 97 | return ' ' |
| 98 | } |
| 99 | return r |
| 100 | }, strings.TrimSpace(s)) |
| 101 | if len(s) > max { |
| 102 | s = s[:max] |
| 103 | for len(s) > 0 && !utf8.ValidString(s) { |
| 104 | s = s[:len(s)-1] |
| 105 | } |
| 106 | } |
| 107 | return s |
| 108 | } |
| 109 | |
| 110 | // DomainOf extracts the lowercase domain from an address, tolerating a |
| 111 | // display-name wrapper. Empty when there is no domain to find. Strips one |
| 112 | // trailing dot and any stray angle bracket. |
| 113 | func DomainOf(address string) string { |
| 114 | s := strings.TrimSpace(address) |
| 115 | if i := strings.LastIndex(s, "<"); i >= 0 { |
| 116 | if j := strings.LastIndex(s, ">"); j > i { |
| 117 | s = s[i+1 : j] |
| 118 | } |
| 119 | } |
| 120 | at := strings.LastIndex(s, "@") |
| 121 | if at < 0 || at == len(s)-1 { |
| 122 | return "" |
| 123 | } |
| 124 | dom := strings.TrimSpace(s[at+1:]) |
| 125 | // Strip stray trailing > from unbalanced brackets |
| 126 | if len(dom) > 0 && dom[len(dom)-1] == '>' { |
| 127 | dom = strings.TrimRight(dom, ">") |
| 128 | if dom == "" { |
| 129 | return "" |
| 130 | } |
| 131 | } |
| 132 | // Strip exactly one trailing dot (FQDN normalisation) |
| 133 | if len(dom) > 1 && dom[len(dom)-1] == '.' { |
| 134 | dom = dom[:len(dom)-1] |
| 135 | } |
| 136 | return strings.ToLower(dom) |
| 137 | } |
| 138 | |
| 139 | // ParseEvent reads one SES notification. |
| 140 | // |
| 141 | // An UNKNOWN event type parses successfully and is carried through as-is. |
| 142 | // SES adds event types, and refusing one would make the ingest endpoint |
| 143 | // answer 500, which makes SNS retry it forever — a self-inflicted flood |
| 144 | // over a field nothing acts on. |
| 145 | // |
| 146 | // A missing source IS refused: an event that can be attributed to no tenant |
| 147 | // is worse than no event, because it would be silently dropped from the |
| 148 | // counts that enforce a cap. |
| 149 | func ParseEvent(body []byte) (Event, error) { |
| 150 | // Two reads of the same body, deliberately. This one is everything the |
| 151 | // counts and the cap depend on, typed exactly as it was before the log |
| 152 | // existed, and it is the only one allowed to refuse the event: a refused |
| 153 | // event is a count silently dropped. |
| 154 | var raw struct { |
| 155 | EventType string `json:"eventType"` |
| 156 | Mail struct { |
| 157 | Timestamp string `json:"timestamp"` |
| 158 | MessageID string `json:"messageId"` |
| 159 | Source string `json:"source"` |
| 160 | } `json:"mail"` |
| 161 | } |
| 162 | if err := json.Unmarshal(body, &raw); err != nil { |
| 163 | return Event{}, fmt.Errorf("mail: parse ses event: %w", err) |
| 164 | } |
| 165 | if raw.Mail.Source == "" { |
| 166 | return Event{}, fmt.Errorf("mail: ses event has no mail.source — it can be attributed to no tenant") |
| 167 | } |
| 168 | // Refuse control characters on the raw source: an injected newline anywhere |
| 169 | // in the field indicates malformation and must not be trusted. |
| 170 | for _, c := range raw.Mail.Source { |
| 171 | if c < 32 || c == 127 { |
| 172 | return Event{}, fmt.Errorf("mail: ses event source contains control character (0x%02x)", c) |
| 173 | } |
| 174 | } |
| 175 | // Extract the address part (stripping display-name wrapper if present) and |
| 176 | // test it for ambiguity. A single envelope MAIL FROM must extract to exactly |
| 177 | // one @ and no commas. |
| 178 | extractedAddr := raw.Mail.Source |
| 179 | s := strings.TrimSpace(raw.Mail.Source) |
| 180 | if i := strings.LastIndex(s, "<"); i >= 0 { |
| 181 | if j := strings.LastIndex(s, ">"); j > i { |
| 182 | // Has a bracketed span; refuse if there are multiple bracket pairs |
| 183 | // (indicated by another < before the final one). |
| 184 | if strings.Contains(s[:i], "<") { |
| 185 | return Event{}, fmt.Errorf("mail: ses event source %q is ambiguous — multiple bracketed address spans", raw.Mail.Source) |
| 186 | } |
| 187 | extractedAddr = strings.TrimSpace(s[i+1 : j]) |
| 188 | } |
| 189 | } |
| 190 | // Check the extracted address for ambiguity: commas or multiple @ symbols. |
| 191 | // Note: "a@b"@example.com (a quoted local part containing @) is refused here. |
| 192 | // This is deliberate: SES mail.source is a bare envelope MAIL FROM with no |
| 193 | // display name, and quoted local parts with @ essentially never appear there. |
| 194 | // Quote-aware parsing is not worth the machinery for a case that does not occur. |
| 195 | if strings.Contains(extractedAddr, ",") || strings.Count(extractedAddr, "@") > 1 { |
| 196 | return Event{}, fmt.Errorf("mail: ses event source %q is ambiguous — cannot reliably attribute to one tenant", raw.Mail.Source) |
| 197 | } |
| 198 | dom := DomainOf(raw.Mail.Source) |
| 199 | if dom == "" { |
| 200 | return Event{}, fmt.Errorf("mail: ses event source %q has no domain", raw.Mail.Source) |
| 201 | } |
| 202 | ev := Event{ |
| 203 | Type: raw.EventType, Domain: dom, |
| 204 | From: raw.Mail.Source, MessageID: raw.Mail.MessageID, |
| 205 | } |
| 206 | if raw.Mail.Timestamp != "" { |
| 207 | if at, err := time.Parse(time.RFC3339, raw.Mail.Timestamp); err == nil { |
| 208 | ev.SentAt = at |
| 209 | ev.At = at |
| 210 | } |
| 211 | } |
| 212 | ev.MessageID = cleanField(ev.MessageID, maxDiagnosticLen) |
| 213 | readLogFields(body, raw.EventType, &ev) |
| 214 | return ev, nil |
| 215 | } |
| 216 | |
| 217 | // readLogFields is the second read: what only the per-message log uses — |
| 218 | // who the message went to, who sent it, and what the receiving server said. |
| 219 | // A field here in a shape this code does not expect is not a reason to lose |
| 220 | // the event, so nothing here refuses it: the part that does not read is left |
| 221 | // empty, the reason is kept in ev.Incomplete for the caller to log, and the |
| 222 | // count goes ahead. The envelope and the event's own block are read |
| 223 | // separately, so a malformed bounce block does not also cost the sender. |
| 224 | func readLogFields(body []byte, eventType string, ev *Event) { |
| 225 | type recipient struct { |
| 226 | EmailAddress string `json:"emailAddress"` |
| 227 | Status string `json:"status"` |
| 228 | DiagnosticCode string `json:"diagnosticCode"` |
| 229 | } |
| 230 | var problems []string |
| 231 | var envelope struct { |
| 232 | Mail struct { |
| 233 | Destination []string `json:"destination"` |
| 234 | Tags map[string][]string `json:"tags"` |
| 235 | } `json:"mail"` |
| 236 | } |
| 237 | if err := json.Unmarshal(body, &envelope); err != nil { |
| 238 | problems = append(problems, "mail: "+err.Error()) |
| 239 | } |
| 240 | if ids := envelope.Mail.Tags["ses:caller-identity"]; len(ids) > 0 { |
| 241 | ev.Caller = cleanField(ids[0], maxDiagnosticLen) |
| 242 | } |
| 243 | for _, to := range envelope.Mail.Destination { |
| 244 | if to = cleanField(to, maxAddressLen); to != "" { |
| 245 | ev.Destination = append(ev.Destination, to) |
| 246 | } |
| 247 | } |
| 248 | |
| 249 | // Each type carries its own block; the others are simply empty. Read |
| 250 | // only the one this event is, so a malformed message that carries two |
| 251 | // cannot have one type's recipients reported under another's name. |
| 252 | var raw struct { |
| 253 | Bounce struct { |
| 254 | Timestamp string `json:"timestamp"` |
| 255 | BounceType string `json:"bounceType"` |
| 256 | BounceSubType string `json:"bounceSubType"` |
| 257 | BouncedRecipients []recipient `json:"bouncedRecipients"` |
| 258 | } `json:"bounce"` |
| 259 | Complaint struct { |
| 260 | Timestamp string `json:"timestamp"` |
| 261 | ComplaintFeedbackType string `json:"complaintFeedbackType"` |
| 262 | ComplainedRecipients []recipient `json:"complainedRecipients"` |
| 263 | } `json:"complaint"` |
| 264 | Delivery struct { |
| 265 | Timestamp string `json:"timestamp"` |
| 266 | Recipients []string `json:"recipients"` |
| 267 | SMTPResponse string `json:"smtpResponse"` |
| 268 | } `json:"delivery"` |
| 269 | Reject struct { |
| 270 | Reason string `json:"reason"` |
| 271 | } `json:"reject"` |
| 272 | DeliveryDelay struct { |
| 273 | Timestamp string `json:"timestamp"` |
| 274 | DelayType string `json:"delayType"` |
| 275 | DelayedRecipients []recipient `json:"delayedRecipients"` |
| 276 | } `json:"deliveryDelay"` |
| 277 | } |
| 278 | block := map[string]string{ |
| 279 | EventBounce: "bounce", EventComplaint: "complaint", EventDelivery: "delivery", |
| 280 | EventReject: "reject", EventDeliveryDelay: "deliveryDelay", |
| 281 | }[eventType] |
| 282 | if block != "" { |
| 283 | var blocks map[string]json.RawMessage |
| 284 | if err := json.Unmarshal(body, &blocks); err == nil { |
| 285 | // Wrap the one block back under its own key and read only that. |
| 286 | if one, ok := blocks[block]; ok { |
| 287 | if err := json.Unmarshal([]byte(`{"`+block+`":`+string(one)+`}`), &raw); err != nil { |
| 288 | problems = append(problems, block+": "+err.Error()) |
| 289 | } |
| 290 | } |
| 291 | } |
| 292 | } |
| 293 | if len(problems) > 0 { |
| 294 | ev.Incomplete = cleanField(strings.Join(problems, "; "), maxDiagnosticLen) |
| 295 | } |
| 296 | |
| 297 | // When the thing this event reports happened, where SES says: a bounce |
| 298 | // or a delay can arrive a day after the send, and an event dated by the |
| 299 | // send is both wrong on a screen and invisible to "since yesterday". |
| 300 | eventTime := map[string]string{ |
| 301 | EventBounce: raw.Bounce.Timestamp, EventComplaint: raw.Complaint.Timestamp, |
| 302 | EventDelivery: raw.Delivery.Timestamp, EventDeliveryDelay: raw.DeliveryDelay.Timestamp, |
| 303 | }[eventType] |
| 304 | if eventTime != "" { |
| 305 | if at, err := time.Parse(time.RFC3339, eventTime); err == nil { |
| 306 | ev.At = at |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | outcomes := func(rs []recipient) { |
| 311 | for _, r := range rs { |
| 312 | to := cleanField(r.EmailAddress, maxAddressLen) |
| 313 | if to == "" { |
| 314 | continue |
| 315 | } |
| 316 | ev.Outcomes = append(ev.Outcomes, Outcome{Recipient: to, |
| 317 | Status: cleanField(r.Status, maxDetailLen), Diagnostic: cleanField(r.DiagnosticCode, maxDiagnosticLen)}) |
| 318 | } |
| 319 | } |
| 320 | switch eventType { |
| 321 | case EventBounce: |
| 322 | ev.Detail.BounceType = cleanField(raw.Bounce.BounceType, maxDetailLen) |
| 323 | ev.Detail.BounceSubType = cleanField(raw.Bounce.BounceSubType, maxDetailLen) |
| 324 | outcomes(raw.Bounce.BouncedRecipients) |
| 325 | case EventComplaint: |
| 326 | ev.Detail.ComplaintType = cleanField(raw.Complaint.ComplaintFeedbackType, maxDetailLen) |
| 327 | outcomes(raw.Complaint.ComplainedRecipients) |
| 328 | case EventDelivery: |
| 329 | said := cleanField(raw.Delivery.SMTPResponse, maxDiagnosticLen) |
| 330 | for _, to := range raw.Delivery.Recipients { |
| 331 | if to = cleanField(to, maxAddressLen); to != "" { |
| 332 | ev.Outcomes = append(ev.Outcomes, Outcome{Recipient: to, Diagnostic: said}) |
| 333 | } |
| 334 | } |
| 335 | case EventReject: |
| 336 | ev.Detail.RejectReason = cleanField(raw.Reject.Reason, maxDiagnosticLen) |
| 337 | case EventDeliveryDelay: |
| 338 | ev.Detail.DelayType = cleanField(raw.DeliveryDelay.DelayType, maxDetailLen) |
| 339 | outcomes(raw.DeliveryDelay.DelayedRecipients) |
| 340 | } |
| 341 | } |
| 342 | |
| 343 | // Recipients is who this event is about, one entry per address: the |
| 344 | // recipients it reports an outcome for, or — for an event about the whole |
| 345 | // message, a Send or a Reject — everyone it was addressed to. A per-recipient |
| 346 | // event (a bounce, a delivery, a complaint, a delay) that names nobody is NOT |
| 347 | // widened to every addressee: that would report a bounce for addresses that |
| 348 | // did not bounce. It, like any event that names nobody at all, yields a |
| 349 | // single empty entry, so it is still recorded once rather than not at all. |
| 350 | func (ev Event) Recipients() []Outcome { |
| 351 | if len(ev.Outcomes) > 0 { |
| 352 | return ev.Outcomes |
| 353 | } |
| 354 | wholeMessage := ev.Type == EventSend || ev.Type == EventReject |
| 355 | if len(ev.Destination) == 0 || !wholeMessage { |
| 356 | return []Outcome{{}} |
| 357 | } |
| 358 | out := make([]Outcome, 0, len(ev.Destination)) |
| 359 | for _, to := range ev.Destination { |
| 360 | out = append(out, Outcome{Recipient: to}) |
| 361 | } |
| 362 | return out |
| 363 | } |
| 364 | |
| 365 | // Summary is Detail in a few words, for a table: "Permanent/NoEmail", |
| 366 | // "abuse", "Bad content". Empty when the event has nothing to add. |
| 367 | func (d EventDetail) Summary() string { |
| 368 | switch { |
| 369 | case d.BounceType != "" && d.BounceSubType != "": |
| 370 | return d.BounceType + "/" + d.BounceSubType |
| 371 | case d.BounceType != "": |
| 372 | return d.BounceType |
| 373 | case d.ComplaintType != "": |
| 374 | return d.ComplaintType |
| 375 | case d.RejectReason != "": |
| 376 | return d.RejectReason |
| 377 | } |
| 378 | return d.DelayType |
| 379 | } |
| 380 | |
amadan