amadan

carlos / platform Public

reqrec: redaction and classification for edge request records

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
claude afde4ae0fb547ed2140c847bf3f3a72ce26bbfbf parent ccb7298
4 files changed, +426 −0
  • internal/reqrec/classify.go +97 −0
  • internal/reqrec/classify_test.go +53 −0
  • internal/reqrec/redact.go +211 −0
  • internal/reqrec/redact_test.go +65 −0
diff --git a/internal/reqrec/classify.go b/internal/reqrec/classify.go
new file mode 100644
index 000000000..d177e6855
--- /dev/null
+++ b/internal/reqrec/classify.go
@@ -0,0 +1,97 @@
+package reqrec
+
+import (
+ "crypto/hmac"
+ "crypto/sha256"
+ "encoding/hex"
+ "net/netip"
+ "strings"
+)
+
+var (
+ scannerMarkers = []string{
+ "zgrab", "masscan", "nuclei", "sqlmap", "nikto", "nmap", "censys", "expanse",
+ "go web pentest", "libredtail", "httpx", "wpscan", "gobuster", "dirbuster",
+ "ffuf", "software security research",
+ }
+ toolMarkers = []string{
+ "curl/", "wget/", "python-requests", "python-urllib", "aiohttp", "go-http-client",
+ "okhttp", "java/", "libwww", "node-fetch", "axios/", "httpclient", "scrapy",
+ "postmanruntime", "insomnia",
+ }
+ crawlerMarkers = []string{
+ "bot", "crawler", "spider", "slurp", "facebookexternalhit", "ahrefs", "semrush",
+ "bytespider", "yandex", "baiduspider", "pingdom", "uptimerobot", "statuscake",
+ "lighthouse", "pagespeed",
+ }
+ browserMarkers = []string{"chrome/", "safari/", "firefox/", "edg/"}
+)
+
+func containsAny(s string, markers []string) bool {
+ for _, m := range markers {
+ if strings.Contains(s, m) {
+ return true
+ }
+ }
+ return false
+}
+
+// UAFamily sorts a User-Agent into one coarse family: empty, scanner, tool,
+// crawler, browser or other. The order matters, because scanners often claim
+// to be Mozilla and crawlers often carry a browser's tokens too.
+func UAFamily(ua string) string {
+ ua = strings.ToLower(strings.TrimSpace(ua))
+ switch {
+ case ua == "":
+ return "empty"
+ case containsAny(ua, scannerMarkers):
+ return "scanner"
+ case containsAny(ua, toolMarkers):
+ return "tool"
+ case containsAny(ua, crawlerMarkers):
+ return "crawler"
+ case strings.Contains(ua, "mozilla/5.0") && containsAny(ua, browserMarkers):
+ return "browser"
+ }
+ return "other"
+}
+
+// clientPrefix parses ip, drops any IPv4-in-IPv6 mapping and zone, and returns
+// the network of the given width: v4Bits for IPv4, v6Bits for IPv6. It returns
+// "" for anything that is not an address.
+func clientPrefix(ip string, v4Bits, v6Bits int) string {
+ addr, err := netip.ParseAddr(strings.TrimSpace(ip))
+ if err != nil {
+ return ""
+ }
+ addr = addr.Unmap().WithZone("")
+ bits := v6Bits
+ if addr.Is4() {
+ bits = v4Bits
+ }
+ p, err := addr.Prefix(bits)
+ if err != nil {
+ return ""
+ }
+ return p.String()
+}
+
+// ClientKey is the network a client is tracked by: its /32 for IPv4 and its /64
+// for IPv6, since one IPv6 subscriber holds a whole /64. It is "" when ip is
+// not an address.
+func ClientKey(ip string) string { return clientPrefix(ip, 32, 64) }
+
+// ClientPrefix is the coarse network a record stores in the clear: the /24 for
+// IPv4 and the /48 for IPv6. It is "" when ip is not an address.
+func ClientPrefix(ip string) string { return clientPrefix(ip, 24, 48) }
+
+// ClientHash is a short keyed hash of a ClientKey, so records can be grouped by
+// client without storing the address. It is "" when there is no salt or no key.
+func ClientHash(salt []byte, key string) string {
+ if len(salt) == 0 || key == "" {
+ return ""
+ }
+ mac := hmac.New(sha256.New, salt)
+ mac.Write([]byte(key))
+ return hex.EncodeToString(mac.Sum(nil))[:12]
+}
diff --git a/internal/reqrec/classify_test.go b/internal/reqrec/classify_test.go
new file mode 100644
index 000000000..f0cebfa18
--- /dev/null
+++ b/internal/reqrec/classify_test.go
@@ -0,0 +1,53 @@
+package reqrec
+
+import (
+ "bytes"
+ "testing"
+)
+
+func TestUAFamily(t *testing.T) {
+ cases := map[string]string{
+ "": "empty",
+ "Mozilla/5.0 zgrab/0.x": "scanner",
+ "Expanse, a Palo Alto Networks company, searches across the global IPv4 space": "scanner",
+ "curl/8.7.1": "tool",
+ "python-requests/2.31": "tool",
+ "Go-http-client/1.1": "tool",
+ "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)": "crawler",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36": "browser",
+ "SomethingElse/1.0": "other",
+ }
+ for ua, want := range cases {
+ if got := UAFamily(ua); got != want {
+ t.Errorf("UAFamily(%q) = %q, want %q", ua, got, want)
+ }
+ }
+}
+
+func TestClientKeyAndPrefix(t *testing.T) {
+ cases := []struct{ ip, key, prefix string }{
+ {"203.0.113.7", "203.0.113.7/32", "203.0.113.0/24"},
+ {"::ffff:203.0.113.7", "203.0.113.7/32", "203.0.113.0/24"},
+ {"2001:db8:1234:5678:9abc::1", "2001:db8:1234:5678::/64", "2001:db8:1234::/48"},
+ {"not-an-ip", "", ""},
+ }
+ for _, c := range cases {
+ if got := ClientKey(c.ip); got != c.key {
+ t.Errorf("ClientKey(%q) = %q, want %q", c.ip, got, c.key)
+ }
+ if got := ClientPrefix(c.ip); got != c.prefix {
+ t.Errorf("ClientPrefix(%q) = %q, want %q", c.ip, got, c.prefix)
+ }
+ }
+}
+
+func TestClientHash(t *testing.T) {
+ salt := bytes.Repeat([]byte{7}, 32)
+ a, b := ClientHash(salt, "203.0.113.7/32"), ClientHash(salt, "203.0.113.8/32")
+ if len(a) != 12 || a == b || a != ClientHash(salt, "203.0.113.7/32") {
+ t.Errorf("hash not stable, distinct and 12 hex: %q %q", a, b)
+ }
+ if ClientHash(nil, "203.0.113.7/32") != "" || ClientHash(salt, "") != "" {
+ t.Error("nil salt or empty key must give an empty hash")
+ }
+}
diff --git a/internal/reqrec/redact.go b/internal/reqrec/redact.go
new file mode 100644
index 000000000..3a17945c2
--- /dev/null
+++ b/internal/reqrec/redact.go
@@ -0,0 +1,211 @@
+// Package reqrec holds the pure rules for edge request records: what may be
+// stored about a request, and how a client and a user agent are summarised.
+// A record never carries a password, a token, a cookie value or a full client
+// address, so everything here either masks, drops or coarsens its input.
+// It imports only the standard library so the edge, host and console can all
+// share it.
+package reqrec
+
+import (
+ "net/url"
+ "strings"
+ "unicode/utf8"
+)
+
+const (
+ // MaskToken replaces anything that looks like a secret or an opaque id.
+ MaskToken = "[masked]"
+ // MaxPath is the most bytes of a request path a record keeps.
+ MaxPath = 512
+ // MaxQueryNames is the most bytes of the comma-joined parameter names a
+ // record keeps.
+ MaxQueryNames = 256
+ // MaxFragment is the most bytes of one evidence fragment a record keeps.
+ MaxFragment = 64
+)
+
+// minRun is the shortest run that can be a token or an id. Hyphenated slugs
+// are split at the hyphens into short words, so they never reach it.
+const minRun = 24
+
+func isRunByte(c byte) bool {
+ return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '+' || c == '='
+}
+
+func isHexByte(c byte) bool {
+ return c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F'
+}
+
+// maskable reports whether a maximal run of [A-Za-z0-9+=] is long enough and
+// mixed enough to be a secret: either it holds a letter and a digit, or it is
+// all hex digits (a long digit string or a hash with no digit in it).
+func maskable(run string) bool {
+ if len(run) < minRun {
+ return false
+ }
+ letter, digit, hex := false, false, true
+ for i := 0; i < len(run); i++ {
+ c := run[i]
+ switch {
+ case c >= '0' && c <= '9':
+ digit = true
+ case c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z':
+ letter = true
+ }
+ if !isHexByte(c) {
+ hex = false
+ }
+ }
+ return letter && digit || hex
+}
+
+// maskRuns replaces every maskable run in s with MaskToken. Any byte outside
+// [A-Za-z0-9+=] ends a run, which is what keeps hyphenated slugs and file
+// extensions out of it.
+func maskRuns(s string) string {
+ var b strings.Builder
+ last := 0
+ for i := 0; i < len(s); {
+ if !isRunByte(s[i]) {
+ i++
+ continue
+ }
+ j := i
+ for j < len(s) && isRunByte(s[j]) {
+ j++
+ }
+ if maskable(s[i:j]) {
+ b.WriteString(s[last:i])
+ b.WriteString(MaskToken)
+ last = j
+ }
+ i = j
+ }
+ if last == 0 {
+ return s
+ }
+ b.WriteString(s[last:])
+ return b.String()
+}
+
+// truncate cuts s to at most n bytes without splitting a rune. s must already
+// be valid UTF-8.
+func truncate(s string, n int) string {
+ if len(s) <= n {
+ return s
+ }
+ for n > 0 && !utf8.RuneStart(s[n]) {
+ n--
+ }
+ return s[:n]
+}
+
+// clean makes s valid UTF-8, masks secret-looking runs and caps the length.
+func clean(s string, max int) string {
+ s = strings.ToValidUTF8(s, "�")
+ return truncate(maskRuns(s), max)
+}
+
+// MaskPath returns a request path that is safe to store: valid UTF-8, with
+// token-like runs masked, and at most MaxPath bytes.
+func MaskPath(p string) string {
+ return clean(p, MaxPath)
+}
+
+// QueryNames returns the parameter names of a raw query string, comma-joined,
+// in first-seen order and without repeats. Values are never kept. A name that
+// would push the result past MaxQueryNames bytes is dropped whole.
+func QueryNames(rawQuery string) string {
+ if rawQuery == "" {
+ return ""
+ }
+ seen := map[string]bool{}
+ var out strings.Builder
+ for _, pair := range strings.FieldsFunc(rawQuery, func(r rune) bool { return r == '&' || r == ';' }) {
+ name, _, _ := strings.Cut(pair, "=")
+ if un, err := url.QueryUnescape(name); err == nil {
+ name = un
+ }
+ name = strings.ToValidUTF8(name, "�")
+ name = maskRuns(name)
+ if name == "" || seen[name] {
+ continue
+ }
+ seen[name] = true
+ need := len(name)
+ if out.Len() > 0 {
+ need++
+ }
+ if out.Len()+need > MaxQueryNames {
+ continue
+ }
+ if out.Len() > 0 {
+ out.WriteByte(',')
+ }
+ out.WriteString(name)
+ }
+ return out.String()
+}
+
+// Source says where a fragment of evidence came from.
+type Source string
+
+const (
+ SourcePath Source = "path"
+ SourceQuery Source = "query"
+ SourceHeader Source = "header"
+ SourceBody Source = "body"
+ SourceCookie Source = "cookie"
+)
+
+// safeHeaders are the only request headers whose values a record may keep.
+var safeHeaders = []string{"User-Agent", "Referer", "Content-Type", "Accept", "Accept-Language", "Accept-Encoding"}
+
+// secretNames are the substrings that mark a parameter name as carrying a
+// secret. The match is deliberately broad: dropping a harmless value costs a
+// little evidence, keeping a secret costs much more.
+var secretNames = []string{
+ "password", "passwd", "pwd", "token", "secret", "key", "session",
+ "auth", "sig", "signature", "credential", "otp", "code",
+}
+
+// IsSecretName reports whether a parameter or field name looks like it holds a
+// secret, by case-insensitive substring match.
+func IsSecretName(name string) bool {
+ lower := strings.ToLower(name)
+ for _, s := range secretNames {
+ if strings.Contains(lower, s) {
+ return true
+ }
+ }
+ return false
+}
+
+// Fragment returns the part of a request value a record may keep as evidence,
+// and whether it may be kept at all. Cookies are never kept, headers only from
+// the safe list, and query and body values only when the name is not secret.
+// Anything kept is run-masked and capped at MaxFragment bytes.
+func Fragment(src Source, name, value string) (string, bool) {
+ switch src {
+ case SourcePath:
+ case SourceQuery, SourceBody:
+ if IsSecretName(name) {
+ return "", false
+ }
+ case SourceHeader:
+ ok := false
+ for _, h := range safeHeaders {
+ if strings.EqualFold(name, h) {
+ ok = true
+ break
+ }
+ }
+ if !ok {
+ return "", false
+ }
+ default:
+ // Cookies, and any source this code does not know about.
+ return "", false
+ }
+ return clean(value, MaxFragment), true
+}
diff --git a/internal/reqrec/redact_test.go b/internal/reqrec/redact_test.go
new file mode 100644
index 000000000..d0478f1ee
--- /dev/null
+++ b/internal/reqrec/redact_test.go
@@ -0,0 +1,65 @@
+package reqrec
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
+
+func TestMaskPath(t *testing.T) {
+ cases := map[string]string{
+ "/en/articles/how-do-i-access-the-admin-pane-2023": "/en/articles/how-do-i-access-the-admin-pane-2023",
+ "/reset/3f9a0c1d2e4b5a6978877665544332211": "/reset/[masked]",
+ "/assets/admin/application-0123456789abcdef0123456789abcdef.js": "/assets/admin/application-[masked].js",
+ "/i/eyJhbGciOiJIUzI1NiJ9xQ2w9pLmNb7Ks4Ua/x.png": "/i/[masked]/x.png",
+ "/wp-login.php": "/wp-login.php",
+ "/short1234abcd": "/short1234abcd",
+ }
+ for in, want := range cases {
+ if got := MaskPath(in); got != want {
+ t.Errorf("MaskPath(%q) = %q, want %q", in, got, want)
+ }
+ }
+ long := "/" + strings.Repeat("a/", 600)
+ if got := MaskPath(long); len(got) > MaxPath || !utf8.ValidString(got) {
+ t.Errorf("long path not truncated to a valid string of at most %d bytes", MaxPath)
+ }
+ if got := MaskPath("/\xff\xfe/x"); !utf8.ValidString(got) {
+ t.Errorf("invalid UTF-8 survived: %q", got)
+ }
+}
+
+func TestQueryNames(t *testing.T) {
+ if got := QueryNames("q=reports'+UNION+ALL&page=2&q=x&token=abc"); got != "q,page,token" {
+ t.Errorf("got %q", got)
+ }
+ if got := QueryNames(""); got != "" {
+ t.Errorf("got %q", got)
+ }
+ many := strings.Repeat("abcdefgh=1&", 2000)
+ if got := QueryNames(many); len(got) > MaxQueryNames {
+ t.Errorf("not capped: %d bytes", len(got))
+ }
+}
+
+func TestFragment(t *testing.T) {
+ if _, ok := Fragment(SourceCookie, "sid", "x"); ok {
+ t.Error("cookie fragment kept")
+ }
+ if _, ok := Fragment(SourceHeader, "Authorization", "Bearer x"); ok {
+ t.Error("authorization kept")
+ }
+ if _, ok := Fragment(SourceHeader, "X-Api-Key", "x"); ok {
+ t.Error("header off the safe list kept")
+ }
+ if _, ok := Fragment(SourceQuery, "reset_token", "x"); ok {
+ t.Error("secret-named parameter kept")
+ }
+ got, ok := Fragment(SourceQuery, "q", "reports' UNION ALL SELECT "+strings.Repeat("N", 100))
+ if !ok || len(got) > MaxFragment || !strings.HasPrefix(got, "reports' UNION ALL SELECT") {
+ t.Errorf("query fragment = %q, %v", got, ok)
+ }
+ if got, ok := Fragment(SourceHeader, "User-Agent", "sqlmap/1.7"); !ok || got != "sqlmap/1.7" {
+ t.Errorf("user-agent fragment = %q, %v", got, ok)
+ }
+}