| 1 | // Edge response compression: one gzip ResponseWriter wrapper applied at the |
| 2 | // client-write boundary in Edge.ServeHTTP, so every serving class — cache |
| 3 | // hits, proxied misses, static trees, the status host — compresses through a |
| 4 | // single site. Apps ship identity bytes; transport is the platform's problem. |
| 5 | // |
| 6 | // The wrapper sits BELOW recordingResponseWriter and ABOVE countingWriter: |
| 7 | // the cache keeps storing identity bodies (one variant, compressed after |
| 8 | // retrieval on every delivery) and the usage meter counts what actually |
| 9 | // crossed the wire. |
| 10 | package edge |
| 11 | |
| 12 | import ( |
| 13 | "bufio" |
| 14 | "compress/gzip" |
| 15 | "fmt" |
| 16 | "io" |
| 17 | "net" |
| 18 | "net/http" |
| 19 | "net/textproto" |
| 20 | "os" |
| 21 | "strconv" |
| 22 | "strings" |
| 23 | "sync" |
| 24 | ) |
| 25 | |
| 26 | // acceptsGzip reports whether r's Accept-Encoding admits gzip. Parsed as the |
| 27 | // q-valued token list RFC 9110 §12.5.3 defines rather than a substring search: |
| 28 | // "gzip;q=0" is a client REFUSING gzip, and answering it with a compressed |
| 29 | // body is a broken response, not a missed optimization. |
| 30 | // |
| 31 | // A specific token outranks the wildcard, so the first gzip token decides — |
| 32 | // "gzip;q=0, *" is a refusal. The wildcard only speaks when gzip is unnamed. |
| 33 | // |
| 34 | // Read through joinHeader, not Header.Get: a field may legally arrive on |
| 35 | // several lines, and Get returns only the first (the same class of bug |
| 36 | // joinHeader's doc comment describes for Vary and Cache-Control). |
| 37 | func acceptsGzip(h http.Header) bool { |
| 38 | var wildcard, wildcardOK bool |
| 39 | for _, part := range strings.Split(joinHeader(h, "Accept-Encoding"), ",") { |
| 40 | token, params, _ := strings.Cut(strings.TrimSpace(part), ";") |
| 41 | switch strings.ToLower(strings.TrimSpace(token)) { |
| 42 | case "gzip", "x-gzip": |
| 43 | return qValue(params) > 0 |
| 44 | case "*": |
| 45 | wildcard, wildcardOK = true, qValue(params) > 0 |
| 46 | } |
| 47 | } |
| 48 | if wildcard { |
| 49 | return wildcardOK |
| 50 | } |
| 51 | return false |
| 52 | } |
| 53 | |
| 54 | // qValue reads the q parameter out of one Accept-Encoding element's parameter |
| 55 | // string ("q=0.5", "q=0"). Absent or malformed means q=1: a client whose |
| 56 | // header we cannot fully parse is treated as accepting, which is the same |
| 57 | // default the stdlib's own negotiation takes. |
| 58 | func qValue(params string) float64 { |
| 59 | for _, p := range strings.Split(params, ";") { |
| 60 | name, val, ok := strings.Cut(strings.TrimSpace(p), "=") |
| 61 | if !ok || !strings.EqualFold(strings.TrimSpace(name), "q") { |
| 62 | continue |
| 63 | } |
| 64 | f, err := strconv.ParseFloat(strings.TrimSpace(val), 64) |
| 65 | if err != nil { |
| 66 | return 1 |
| 67 | } |
| 68 | return f |
| 69 | } |
| 70 | return 1 |
| 71 | } |
| 72 | |
| 73 | // minCompressSize is the Content-Length below which compressing is not worth |
| 74 | // the CPU: gzip's own header and trailer are 18 bytes, and a sub-kilobyte |
| 75 | // response often lands in one packet either way. A response whose length is |
| 76 | // UNKNOWN (streamed, chunked) is compressed regardless — refusing those would |
| 77 | // exempt most proxied HTML, which is the bulk of what there is to save. |
| 78 | const minCompressSize = 1024 |
| 79 | |
| 80 | // compressible reports whether a response with these headers and this status |
| 81 | // should be gzipped. The request-side questions — does the client accept |
| 82 | // gzip, is this a HEAD — are settled once at wrapper construction |
| 83 | // (newGzipWriter), so this reads the response alone. |
| 84 | // |
| 85 | // Every false below is a response that would be WRONG or pointless to |
| 86 | // compress, not merely unprofitable: |
| 87 | // |
| 88 | // - 204/205 have no body; 304 has none and must keep the validators of the |
| 89 | // representation it refers to; 206 is a byte range of a larger body, and |
| 90 | // compressing a range makes the offsets it names meaningless. |
| 91 | // - An existing Content-Encoding means the origin already encoded it. |
| 92 | // Compressing again produces a body no client can decode without knowing |
| 93 | // we did it. |
| 94 | // - Cache-Control: no-transform is the origin forbidding exactly this |
| 95 | // (RFC 9110 §5.5). |
| 96 | // |
| 97 | // A MISSING Content-Type is refused rather than sniffed. Sniffing is the |
| 98 | // stdlib's job and it runs on the bytes we would have already compressed — |
| 99 | // it would read gzip's magic number and label the response application/x-gzip. |
| 100 | func compressible(h http.Header, status int) bool { |
| 101 | switch status { |
| 102 | case http.StatusNoContent, http.StatusResetContent, |
| 103 | http.StatusPartialContent, http.StatusNotModified: |
| 104 | return false |
| 105 | } |
| 106 | if h.Get("Content-Range") != "" { |
| 107 | return false |
| 108 | } |
| 109 | if h.Get("Content-Encoding") != "" { |
| 110 | return false |
| 111 | } |
| 112 | if hasCCDirective(joinHeader(h, "Cache-Control"), "no-transform") { |
| 113 | return false |
| 114 | } |
| 115 | if n, err := strconv.Atoi(h.Get("Content-Length")); err == nil && n < minCompressSize { |
| 116 | return false |
| 117 | } |
| 118 | return compressibleType(h.Get("Content-Type")) |
| 119 | } |
| 120 | |
| 121 | // compressibleType reports whether a media type is text underneath. text/* is |
| 122 | // the bulk of it; the structured suffixes (+json, +xml) catch types nobody has |
| 123 | // enumerated yet, which is why they are here rather than a longer switch. |
| 124 | // |
| 125 | // text/event-stream is the one text type excluded: it is an open stream a |
| 126 | // client consumes event by event, and buffering it inside a compressor is a |
| 127 | // hang, not a saving. |
| 128 | func compressibleType(ct string) bool { |
| 129 | mt, _, _ := strings.Cut(ct, ";") |
| 130 | mt = strings.ToLower(strings.TrimSpace(mt)) |
| 131 | if mt == "" || mt == "text/event-stream" { |
| 132 | return false |
| 133 | } |
| 134 | if strings.HasPrefix(mt, "text/") || |
| 135 | strings.HasSuffix(mt, "+json") || strings.HasSuffix(mt, "+xml") { |
| 136 | return true |
| 137 | } |
| 138 | switch mt { |
| 139 | case "application/json", "application/javascript", "application/x-javascript", |
| 140 | "application/xml", "application/wasm": |
| 141 | return true |
| 142 | } |
| 143 | return false |
| 144 | } |
| 145 | |
| 146 | // weakenETag downgrades a strong validator to a weak one, in place. A gzipped |
| 147 | // body is a different representation of the same resource, so an ETag that |
| 148 | // claims byte-for-byte equality with the identity body is a lie — and the one |
| 149 | // thing that lie breaks is a Range request served against the wrong bytes. |
| 150 | // Conditional requests keep working: If-None-Match compares weakly for GET, |
| 151 | // so a client returning W/"abc" still matches the origin's "abc" and gets its |
| 152 | // 304. |
| 153 | func weakenETag(h http.Header) { |
| 154 | if et := h.Get("ETag"); strings.HasPrefix(et, `"`) { |
| 155 | h.Set("ETag", `W/`+et) |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | // gzipPool recycles compressors. We compress on every delivery, cache hits |
| 160 | // included, so a fresh gzip.Writer per response would be a steady stream of |
| 161 | // garbage on a busy edge for no reason. |
| 162 | var gzipPool = sync.Pool{New: func() any { |
| 163 | w, _ := gzip.NewWriterLevel(io.Discard, gzip.DefaultCompression) |
| 164 | return w |
| 165 | }} |
| 166 | |
| 167 | // gzipWriter compresses an eligible response on its way to the client. |
| 168 | // |
| 169 | // It owns a PRIVATE header map. Every other ResponseWriter wrapper in this |
| 170 | // package embeds the writer below, so Header() returns one shared map from the |
| 171 | // handler all the way down to the connection — and maybeStore caches |
| 172 | // rec.Header().Clone(). Were the compression headers set on that shared map, |
| 173 | // the cache would store an identity body labelled Content-Encoding: gzip and |
| 174 | // hand it to every later visitor. So callers above read and write hdr, and the |
| 175 | // edits that describe the COMPRESSED representation go on the real writer's |
| 176 | // map alone. |
| 177 | // |
| 178 | // Deliberately does NOT implement io.ReaderFrom: io.Copy (and so |
| 179 | // http.ServeFile) falls back to Write, which is what compresses. |
| 180 | type gzipWriter struct { |
| 181 | http.ResponseWriter |
| 182 | hdr http.Header |
| 183 | gz *gzip.Writer // nil unless this response is being compressed |
| 184 | wroteHdr bool |
| 185 | } |
| 186 | |
| 187 | // newGzipWriter returns a wrapper for r, or nil when this request can never |
| 188 | // produce a compressed response — no wrapper at all, so the writer stack is |
| 189 | // exactly what it is today. The request-side questions are settled here, once: |
| 190 | // everything after depends only on the response. |
| 191 | // |
| 192 | // HEAD is excluded rather than compressed-with-no-body: there is nothing to |
| 193 | // compress, and a HEAD's truthful Content-Length is the entire value of the |
| 194 | // response. The accepted cost is that a HEAD advertises the identity length |
| 195 | // while a GET of the same URL returns gzip with no length at all. |
| 196 | func newGzipWriter(w http.ResponseWriter, r *http.Request) *gzipWriter { |
| 197 | if r.Method == http.MethodHead || !acceptsGzip(r.Header) { |
| 198 | return nil |
| 199 | } |
| 200 | return &gzipWriter{ResponseWriter: w, hdr: http.Header{}} |
| 201 | } |
| 202 | |
| 203 | // Header returns the PRIVATE map. See the type's doc comment: this is what |
| 204 | // keeps the response cache storing identity bodies. |
| 205 | func (g *gzipWriter) Header() http.Header { return g.hdr } |
| 206 | |
| 207 | // syncHeader replaces the real writer's header block with the private one. |
| 208 | // Wholesale, not a merge: a handler that DELETED a header between blocks (the |
| 209 | // shape the purge-on-103 test exercises) must see that deletion reach the |
| 210 | // client, which a merge would silently undo. |
| 211 | func (g *gzipWriter) syncHeader() { |
| 212 | down := g.ResponseWriter.Header() |
| 213 | for k := range down { |
| 214 | delete(down, k) |
| 215 | } |
| 216 | for k, v := range g.hdr { |
| 217 | down[k] = v |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | func (g *gzipWriter) WriteHeader(status int) { |
| 222 | // 1xx blocks (103 Early Hints, forwarded by httputil.ReverseProxy) precede |
| 223 | // the final status. Sync and forward without latching — the same shape |
| 224 | // recordingResponseWriter.WriteHeader uses, for the same reason: latching |
| 225 | // one would decide compression on a status that has no body. |
| 226 | if status < 200 { |
| 227 | g.syncHeader() |
| 228 | g.ResponseWriter.WriteHeader(status) |
| 229 | return |
| 230 | } |
| 231 | if g.wroteHdr { |
| 232 | g.ResponseWriter.WriteHeader(status) // superfluous; let the stdlib log it |
| 233 | return |
| 234 | } |
| 235 | g.wroteHdr = true |
| 236 | g.syncHeader() |
| 237 | if compressible(g.hdr, status) { |
| 238 | h := g.ResponseWriter.Header() |
| 239 | h.Set("Content-Encoding", "gzip") |
| 240 | h.Del("Content-Length") // it described the identity body |
| 241 | h.Add("Vary", "Accept-Encoding") |
| 242 | weakenETag(h) |
| 243 | g.gz = gzipPool.Get().(*gzip.Writer) |
| 244 | g.gz.Reset(g.ResponseWriter) |
| 245 | } |
| 246 | g.ResponseWriter.WriteHeader(status) |
| 247 | } |
| 248 | |
| 249 | func (g *gzipWriter) Write(p []byte) (int, error) { |
| 250 | if !g.wroteHdr { |
| 251 | g.WriteHeader(http.StatusOK) |
| 252 | } |
| 253 | if g.gz != nil { |
| 254 | return g.gz.Write(p) |
| 255 | } |
| 256 | return g.ResponseWriter.Write(p) |
| 257 | } |
| 258 | |
| 259 | // finish closes the compressor and returns it to the pool. ServeHTTP defers |
| 260 | // it, AFTER the usage meter's defer, so LIFO runs this one first and the |
| 261 | // metered byte count includes gzip's trailer. |
| 262 | // |
| 263 | // The no-header case is not a no-op: a handler that returned without ever |
| 264 | // writing set its headers on the private map, and nothing has copied them |
| 265 | // down. Without this sync the stdlib emits a bare 200 with none of them. |
| 266 | func (g *gzipWriter) finish() { |
| 267 | if !g.wroteHdr { |
| 268 | g.syncHeader() |
| 269 | return |
| 270 | } |
| 271 | g.syncTrailers() |
| 272 | if g.gz == nil { |
| 273 | return |
| 274 | } |
| 275 | g.gz.Close() |
| 276 | g.gz.Reset(io.Discard) // don't pin this request's writer inside the pool |
| 277 | gzipPool.Put(g.gz) |
| 278 | g.gz = nil |
| 279 | } |
| 280 | |
| 281 | // syncTrailers copies LATE additions to the private map down to the real |
| 282 | // writer. net/http moves a header set after WriteHeader into the trailer block |
| 283 | // when it was announced in Trailer (or carries http.TrailerPrefix) — and every |
| 284 | // such Set landed on the private map, where syncHeader has already been and |
| 285 | // gone. Without this the wrapper silently eats trailers, which |
| 286 | // purge_header.go's contract names as a channel apps may legitimately use. |
| 287 | // |
| 288 | // The fields the compression edits own are skipped while compressing: the |
| 289 | // private map holds the IDENTITY representation's versions of them, and |
| 290 | // copying those down would undo the Content-Encoding, restore a Content-Length |
| 291 | // that describes the wrong bytes, drop our Vary, and re-strengthen the ETag. |
| 292 | func (g *gzipWriter) syncTrailers() { |
| 293 | down := g.ResponseWriter.Header() |
| 294 | for k, v := range g.hdr { |
| 295 | if g.gz != nil && compressionOwned(k) { |
| 296 | continue |
| 297 | } |
| 298 | down[k] = v |
| 299 | } |
| 300 | } |
| 301 | |
| 302 | // compressionOwned reports whether a header name is one the compression edits |
| 303 | // control on the real writer's map. It is called with RAW map keys, which are |
| 304 | // canonical — and canonical for ETag is "Etag". Spelling it "ETag" here (the |
| 305 | // form the RFC and Header.Get use, since Get canonicalizes its argument) |
| 306 | // matched nothing, and the identity ETag was copied back over the weakened |
| 307 | // one. TestGzipWriterForwardsTrailers is what holds this. |
| 308 | func compressionOwned(name string) bool { |
| 309 | switch textproto.CanonicalMIMEHeaderKey(name) { |
| 310 | case "Content-Encoding", "Content-Length", "Vary", "Etag": |
| 311 | return true |
| 312 | } |
| 313 | return false |
| 314 | } |
| 315 | |
| 316 | // Flush pushes the compressor's buffered bytes out and then flushes the writer |
| 317 | // below — needed so a streamed response reaches the client incrementally |
| 318 | // instead of buffering until the handler returns. |
| 319 | func (g *gzipWriter) Flush() { |
| 320 | if g.gz != nil { |
| 321 | g.gz.Flush() |
| 322 | } |
| 323 | if f, ok := g.ResponseWriter.(http.Flusher); ok { |
| 324 | f.Flush() |
| 325 | } |
| 326 | } |
| 327 | |
| 328 | // Hijack forwards to the writer below, when it supports it — the same |
| 329 | // optional-interface pattern countingWriter and recordingResponseWriter use. |
| 330 | // ServeHTTP returns on the upgrade path before this wrapper is constructed, so |
| 331 | // this is not expected on the hot path; it exists so the wrapper is never |
| 332 | // itself the reason a hijack fails. |
| 333 | func (g *gzipWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { |
| 334 | hj, ok := g.ResponseWriter.(http.Hijacker) |
| 335 | if !ok { |
| 336 | return nil, nil, fmt.Errorf("edge: underlying ResponseWriter does not support http.Hijacker") |
| 337 | } |
| 338 | return hj.Hijack() |
| 339 | } |
| 340 | |
| 341 | // gzipEnabled reads the CARLOS_EDGE_GZIP kill switch. Compression is ON unless |
| 342 | // the variable names a false value — the rollback path is an env change and a |
| 343 | // restart, not a platform binary roll. Matches the CARLOS_* env idiom in |
| 344 | // certcache.go and dns.go. |
| 345 | func gzipEnabled() bool { |
| 346 | switch strings.ToLower(strings.TrimSpace(os.Getenv("CARLOS_EDGE_GZIP"))) { |
| 347 | case "off", "false", "0", "no": |
| 348 | return false |
| 349 | } |
| 350 | return true |
| 351 | } |
| 352 | |
amadan