| index 341b396f1..661260c7c 100644 |
| --- a/internal/reqrec/redact.go |
| +++ b/internal/reqrec/redact.go |
| @@ -7,6 +7,7 @@ |
| package reqrec |
| |
| import ( |
| + "math/bits" |
| "net/url" |
| "strings" |
| "unicode/utf8" |
| @@ -38,27 +39,30 @@ func isHexByte(c byte) bool { |
| |
| func isCompoundByte(c byte) bool { return isRunByte(c) || c == '-' || c == '_' } |
| |
| -// maskable reports whether a maximal run of [A-Za-z0-9+=] is long enough and |
| -// mixed enough to be a secret: either it holds a letter and a digit, or it is |
| -// all hex digits (a long digit string or a hash with no digit in it). |
| -func maskable(run string) bool { |
| - if len(run) < minRun { |
| - return false |
| - } |
| - letter, digit, hex := false, false, true |
| - for i := 0; i < len(run); i++ { |
| - c := run[i] |
| - switch { |
| - case c >= '0' && c <= '9': |
| - digit = true |
| - case c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z': |
| - letter = true |
| - } |
| - if !isHexByte(c) { |
| - hex = false |
| +// isBase64Byte is the standard base64 alphabet, which has '/' in it. |
| +func isBase64Byte(c byte) bool { return isRunByte(c) || c == '/' } |
| + |
| +// The character classes a run can mix, as bits. |
| +const ( |
| + classUpper = 1 << iota |
| + classLower |
| + classDigit |
| + classAll = classUpper | classLower | classDigit |
| +) |
| + |
| +func classesOf(s string) int { |
| + var c int |
| + for i := 0; i < len(s); i++ { |
| + switch b := s[i]; { |
| + case b >= 'A' && b <= 'Z': |
| + c |= classUpper |
| + case b >= 'a' && b <= 'z': |
| + c |= classLower |
| + case b >= '0' && b <= '9': |
| + c |= classDigit |
| } |
| } |
| - return letter && digit || hex |
| + return c |
| } |
| |
| // maskableCompound reports whether a maximal run of [A-Za-z0-9+=_-] is a |
| @@ -119,14 +123,114 @@ func maskMatching(s string, in func(byte) bool, test func(string) bool) string { |
| return b.String() |
| } |
| |
| -// maskRuns masks secret-looking tokens in s in two passes. First it judges |
| -// each whole compound of [A-Za-z0-9+=_-], which catches UUIDs and base64url |
| -// strings that a - or _ would otherwise split into short pieces. Then it |
| -// judges each run of [A-Za-z0-9+=] on its own, so a long token inside a |
| -// hyphenated name (application-<hash>.js) is masked and the name is not. |
| +// maskSplitTokens masks standard base64 tokens that a '/' has cut into short |
| +// path segments. Within each run of [A-Za-z0-9+=/] it finds the stretches of |
| +// consecutive segments that each mix at least two of upper case, lower case |
| +// and digits, and masks a stretch as one span when, joined, it holds all |
| +// three and at least minRun characters. A segment of one class ("reset", |
| +// "2023") or an empty one ("//") ends a stretch, so the path words around a |
| +// token are kept; Title Case words hold no digit, so a path of them is kept. |
| +func maskSplitTokens(s string) string { |
| + var b strings.Builder |
| + last := 0 |
| + for i := 0; i < len(s); { |
| + if !isBase64Byte(s[i]) { |
| + i++ |
| + continue |
| + } |
| + j := i |
| + for j < len(s) && isBase64Byte(s[j]) { |
| + j++ |
| + } |
| + // The run's segments as [start, end) offsets into s. |
| + var segs [][2]int |
| + for p := i; p <= j; { |
| + q := p |
| + for q < j && s[q] != '/' { |
| + q++ |
| + } |
| + segs = append(segs, [2]int{p, q}) |
| + p = q + 1 |
| + } |
| + mixed := func(k int) (int, bool) { |
| + c := classesOf(s[segs[k][0]:segs[k][1]]) |
| + return c, bits.OnesCount(uint(c)) >= 2 |
| + } |
| + for a := 0; a < len(segs); { |
| + if _, ok := mixed(a); !ok { |
| + a++ |
| + continue |
| + } |
| + z, cls, n := a, 0, 0 |
| + for z < len(segs) { |
| + c, ok := mixed(z) |
| + if !ok { |
| + break |
| + } |
| + cls |= c |
| + n += segs[z][1] - segs[z][0] |
| + z++ |
| + } |
| + if cls == classAll && n >= minRun { |
| + b.WriteString(s[last:segs[a][0]]) |
| + b.WriteString(MaskToken) |
| + last = segs[z-1][1] |
| + } |
| + a = z |
| + } |
| + i = j |
| + } |
| + if last == 0 { |
| + return s |
| + } |
| + b.WriteString(s[last:]) |
| + return b.String() |
| +} |
| + |
| +// maskRuns masks secret-looking tokens in s. The rule leans to masking: a |
| +// masked word costs a little evidence, a kept token is a secret in a log. |
| +// Three passes, in order: |
| +// |
| +// 1. Each compound of [A-Za-z0-9+=_-] is judged whole (maskableCompound), |
| +// which catches UUIDs and base64url strings that a - or _ would |
| +// otherwise split into short pieces. |
| +// 2. Standard base64 split into path segments at '/' is judged joined |
| +// (maskSplitTokens). |
| +// 3. Every run of [A-Za-z0-9+=] of minRun or more characters is masked, |
| +// whatever it holds: mixed case with no digit, digits only, hex beside |
| +// a non-hex letter. Words that long are rare in a path; ids, hashes and |
| +// tokens are common. This also masks a long token inside a hyphenated |
| +// name (application-<hash>.js) and leaves the name. |
| func maskRuns(s string) string { |
| s = maskMatching(s, isCompoundByte, maskableCompound) |
| - return maskMatching(s, isRunByte, maskable) |
| + s = maskSplitTokens(s) |
| + return maskMatching(s, isRunByte, func(run string) bool { return len(run) >= minRun }) |
| +} |
| + |
| +// isControl reports whether r is a C0 control, DEL or a C1 control: the |
| +// characters a terminal acts on rather than prints. |
| +func isControl(r rune) bool { return r < 0x20 || r == 0x7f || r >= 0x80 && r <= 0x9f } |
| + |
| +// escapeControls writes every control character in s as \xNN (its code |
| +// point in hex), so request text in a record cannot forge a log line or |
| +// drive the terminal of whoever prints it. JSON alone does not do this: |
| +// it escapes controls on the way into a chunk and decoding restores them. |
| +func escapeControls(s string) string { |
| + if strings.IndexFunc(s, isControl) < 0 { |
| + return s |
| + } |
| + const hexDigits = "0123456789abcdef" |
| + var b strings.Builder |
| + for _, r := range s { |
| + if isControl(r) { |
| + b.WriteString(`\x`) |
| + b.WriteByte(hexDigits[r>>4]) |
| + b.WriteByte(hexDigits[r&0xf]) |
| + continue |
| + } |
| + b.WriteRune(r) |
| + } |
| + return b.String() |
| } |
| |
| // truncate cuts s to at most n bytes without splitting a rune. s must already |
| @@ -141,21 +245,25 @@ func truncate(s string, n int) string { |
| return s[:n] |
| } |
| |
| -// clean makes s valid UTF-8, masks secret-looking runs and caps the length. |
| +// clean makes s valid UTF-8, masks secret-looking runs, escapes controls and |
| +// caps the length. The cap comes last so escaping cannot push s past it. |
| func clean(s string, max int) string { |
| s = strings.ToValidUTF8(s, "�") |
| - return truncate(maskRuns(s), max) |
| + return truncate(escapeControls(maskRuns(s)), max) |
| } |
| |
| // MaskPath returns a request path that is safe to store: valid UTF-8, with |
| -// token-like runs masked, and at most MaxPath bytes. |
| +// token-like runs masked and controls escaped, and at most MaxPath bytes. |
| func MaskPath(p string) string { |
| return clean(p, MaxPath) |
| } |
| |
| // QueryNames returns the parameter names of a raw query string, comma-joined, |
| -// in first-seen order and without repeats. Values are never kept. A name that |
| -// would push the result past MaxQueryNames bytes is dropped whole. |
| +// in first-seen order and without repeats. Values are never kept. A component |
| +// with no '=' (?hunter2) is left out: it has no name, and what it carries may |
| +// be the value itself. Names are masked and their controls escaped like a |
| +// path. A name that would push the result past MaxQueryNames bytes is dropped |
| +// whole. |
| func QueryNames(rawQuery string) string { |
| if rawQuery == "" { |
| return "" |
| @@ -163,12 +271,15 @@ func QueryNames(rawQuery string) string { |
| seen := map[string]bool{} |
| var out strings.Builder |
| for _, pair := range strings.FieldsFunc(rawQuery, func(r rune) bool { return r == '&' || r == ';' }) { |
| - name, _, _ := strings.Cut(pair, "=") |
| + name, _, ok := strings.Cut(pair, "=") |
| + if !ok { |
| + continue |
| + } |
| if un, err := url.QueryUnescape(name); err == nil { |
| name = un |
| } |
| name = strings.ToValidUTF8(name, "�") |
| - name = maskRuns(name) |
| + name = escapeControls(maskRuns(name)) |
| if name == "" || seen[name] { |
| continue |
| } |
amadan