| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "encoding/json" |
| 6 | "image" |
| 7 | "image/png" |
| 8 | "io" |
| 9 | "net/http" |
| 10 | "net/http/httptest" |
| 11 | "net/url" |
| 12 | "strings" |
| 13 | "testing" |
| 14 | ) |
| 15 | |
| 16 | func newTestFetch(t *testing.T) (*httptest.Server, *fetchServer) { |
| 17 | t.Helper() |
| 18 | g, err := newFetchServer(t.TempDir() + "/fetch.db") |
| 19 | if err != nil { |
| 20 | t.Fatalf("newFetchServer: %v", err) |
| 21 | } |
| 22 | ts := httptest.NewServer(g.handler()) |
| 23 | t.Cleanup(ts.Close) |
| 24 | t.Cleanup(func() { g.db.Close() }) |
| 25 | return ts, g |
| 26 | } |
| 27 | |
| 28 | // The fetch service unfurls anonymously (no token — it has no members to know about), |
| 29 | // rewrites the card's image to its signed /img proxy, and that proxy URL |
| 30 | // actually serves the image bytes back over the fetch service's own origin. |
| 31 | func TestFetchUnfurlAndImageProxy(t *testing.T) { |
| 32 | ts, g := newTestFetch(t) |
| 33 | |
| 34 | allowPrivatePreviewHosts = true |
| 35 | t.Cleanup(func() { allowPrivatePreviewHosts = false }) |
| 36 | |
| 37 | imgBytes := []byte("not-really-a-png-but-served-as-one") |
| 38 | site := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 39 | if r.URL.Path == "/pic.png" { |
| 40 | w.Header().Set("Content-Type", "image/png") |
| 41 | w.Write(imgBytes) |
| 42 | return |
| 43 | } |
| 44 | w.Header().Set("Content-Type", "text/html") |
| 45 | w.Write([]byte(`<head><meta property="og:title" content="Fetched">` + |
| 46 | `<meta property="og:image" content="/pic.png"></head>`)) |
| 47 | })) |
| 48 | t.Cleanup(site.Close) |
| 49 | |
| 50 | r, err := http.Get(ts.URL + "/api/unfurl?url=" + url.QueryEscape(site.URL)) |
| 51 | if err != nil { |
| 52 | t.Fatalf("unfurl: %v", err) |
| 53 | } |
| 54 | // Callers are browsers on OTHER origins (every space is one): without |
| 55 | // this header fetch() can't read the response and clients silently fall |
| 56 | // back to their instance path — the bug that made the client flip a no-op. |
| 57 | if r.Header.Get("Access-Control-Allow-Origin") != "*" { |
| 58 | t.Fatal("missing Access-Control-Allow-Origin — cross-origin clients can't read the service") |
| 59 | } |
| 60 | var pv LinkPreview |
| 61 | json.NewDecoder(r.Body).Decode(&pv) |
| 62 | r.Body.Close() |
| 63 | if !pv.OK || pv.Title != "Fetched" { |
| 64 | t.Fatalf("preview = %+v", pv) |
| 65 | } |
| 66 | // The image is no longer the site's URL — it's the fetch service's signed proxy. |
| 67 | if !strings.HasPrefix(pv.Image, ts.URL+"/img?") { |
| 68 | t.Fatalf("image not proxied: %q", pv.Image) |
| 69 | } |
| 70 | ir, err := http.Get(pv.Image) |
| 71 | if err != nil { |
| 72 | t.Fatalf("proxied image: %v", err) |
| 73 | } |
| 74 | defer ir.Body.Close() |
| 75 | if ir.StatusCode != http.StatusOK || ir.Header.Get("Content-Type") != "image/png" { |
| 76 | t.Fatalf("proxied image: status %d, type %q", ir.StatusCode, ir.Header.Get("Content-Type")) |
| 77 | } |
| 78 | got, err := io.ReadAll(ir.Body) |
| 79 | if err != nil || string(got) != string(imgBytes) { |
| 80 | t.Fatalf("proxied bytes = %q err=%v", got, err) |
| 81 | } |
| 82 | |
| 83 | // Cached: a second unfurl works with the cache row (and stays sealed at |
| 84 | // rest — no plaintext in the stored row). |
| 85 | var urlHash, blob string |
| 86 | if err := g.db.QueryRow("SELECT url_hash, blob FROM fetch_previews").Scan(&urlHash, &blob); err != nil { |
| 87 | t.Fatalf("raw row: %v", err) |
| 88 | } |
| 89 | for _, secret := range []string{site.URL, "Fetched", "pic.png"} { |
| 90 | if strings.Contains(urlHash+blob, secret) { |
| 91 | t.Fatalf("plaintext %q found in the stored row", secret) |
| 92 | } |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | // /img is not an open proxy: an unsigned or tampered URL never fetches. |
| 97 | func TestFetchImageProxyRejectsUnsigned(t *testing.T) { |
| 98 | ts, g := newTestFetch(t) |
| 99 | |
| 100 | // A URL the fetch service never signed: correct encoding, wrong signature. |
| 101 | u := "aHR0cDovL2V4YW1wbGUuY29tL3guanBn" // base64url("http://example.com/x.jpg") |
| 102 | for _, q := range []string{ |
| 103 | "u=" + u + "&s=" + strings.Repeat("00", 32), |
| 104 | "u=" + u, |
| 105 | "u=%%%&s=zz", |
| 106 | } { |
| 107 | r, err := http.Get(ts.URL + "/img?" + q) |
| 108 | if err != nil { |
| 109 | t.Fatalf("img %q: %v", q, err) |
| 110 | } |
| 111 | r.Body.Close() |
| 112 | if r.StatusCode == http.StatusOK { |
| 113 | t.Fatalf("img %q: served without a valid signature", q) |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | // And a signed URL to a non-image is refused after the fetch. |
| 118 | allowPrivatePreviewHosts = true |
| 119 | t.Cleanup(func() { allowPrivatePreviewHosts = false }) |
| 120 | site := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 121 | w.Header().Set("Content-Type", "text/html") |
| 122 | w.Write([]byte("<html>not an image</html>")) |
| 123 | })) |
| 124 | t.Cleanup(site.Close) |
| 125 | req, _ := http.NewRequest("GET", ts.URL+"/api/unfurl", nil) |
| 126 | signed := g.proxyImageURL(&http.Request{Host: strings.TrimPrefix(ts.URL, "http://"), URL: req.URL}, site.URL) |
| 127 | r, err := http.Get(signed) |
| 128 | if err != nil { |
| 129 | t.Fatalf("signed non-image: %v", err) |
| 130 | } |
| 131 | r.Body.Close() |
| 132 | if r.StatusCode != http.StatusBadGateway { |
| 133 | t.Fatalf("signed non-image: want 502, got %d", r.StatusCode) |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | // Language variants cache as separate rows on the fetch service too — same contract |
| 138 | // as the instance (TestUnfurlLanguage), shared-cache edition. |
| 139 | func TestFetchUnfurlLanguageVariants(t *testing.T) { |
| 140 | ts, g := newTestFetch(t) |
| 141 | |
| 142 | allowPrivatePreviewHosts = true |
| 143 | t.Cleanup(func() { allowPrivatePreviewHosts = false }) |
| 144 | |
| 145 | site := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 146 | w.Header().Set("Content-Type", "text/html") |
| 147 | w.Write([]byte(`<head><meta property="og:title" content="lang:` + |
| 148 | r.Header.Get("Accept-Language") + `"></head>`)) |
| 149 | })) |
| 150 | t.Cleanup(site.Close) |
| 151 | |
| 152 | fetch := func(lang string) string { |
| 153 | t.Helper() |
| 154 | req, _ := http.NewRequest("GET", ts.URL+"/api/unfurl?url="+url.QueryEscape(site.URL), nil) |
| 155 | if lang != "" { |
| 156 | req.Header.Set("Accept-Language", lang) |
| 157 | } |
| 158 | r, err := http.DefaultClient.Do(req) |
| 159 | if err != nil { |
| 160 | t.Fatalf("unfurl(%q): %v", lang, err) |
| 161 | } |
| 162 | defer r.Body.Close() |
| 163 | var pv LinkPreview |
| 164 | json.NewDecoder(r.Body).Decode(&pv) |
| 165 | return pv.Title |
| 166 | } |
| 167 | |
| 168 | if got := fetch("fi"); got != "lang:fi" { |
| 169 | t.Fatalf("fi = %q", got) |
| 170 | } |
| 171 | if got := fetch("en-IE,en;q=0.9"); got != "lang:en-IE, en;q=0.8" { |
| 172 | t.Fatalf("en-IE = %q", got) |
| 173 | } |
| 174 | var rows int |
| 175 | g.db.QueryRow("SELECT COUNT(*) FROM fetch_previews").Scan(&rows) |
| 176 | if rows != 2 { |
| 177 | t.Fatalf("cache rows = %d, want 2", rows) |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | // The client flip is env-gated: with LCHAT_FETCH_URL set, the chat page names |
| 182 | // the fetch service in connect-src and hands the origin to shell.js; unset, the page |
| 183 | // is byte-identical to today's (self-host default = local unfurl). |
| 184 | func TestFetchClientFlipInjection(t *testing.T) { |
| 185 | ts, srv := newTestServer(t) |
| 186 | |
| 187 | get := func() (csp, body string) { |
| 188 | t.Helper() |
| 189 | r, err := http.Get(ts.URL + "/") |
| 190 | if err != nil { |
| 191 | t.Fatalf("GET /: %v", err) |
| 192 | } |
| 193 | defer r.Body.Close() |
| 194 | b, _ := io.ReadAll(r.Body) |
| 195 | return r.Header.Get("Content-Security-Policy"), string(b) |
| 196 | } |
| 197 | |
| 198 | csp, body := get() |
| 199 | if strings.Contains(csp, "fetch.example") { |
| 200 | t.Fatalf("CSP names a fetch service with none configured: %s", csp) |
| 201 | } |
| 202 | if !strings.Contains(body, `window.__fetchURL = ""`) { |
| 203 | t.Fatal("page should inject an empty fetch URL when unconfigured") |
| 204 | } |
| 205 | |
| 206 | srv.fetchURL = "https://fetch.example" |
| 207 | csp, body = get() |
| 208 | if !strings.Contains(csp, "connect-src 'self' https://*.giphy.com") || |
| 209 | !strings.Contains(csp, " https://fetch.example; ") { |
| 210 | t.Fatalf("CSP missing the fetch service origin: %s", csp) |
| 211 | } |
| 212 | if !strings.Contains(body, `window.__fetchURL = "https://fetch.example"`) { |
| 213 | t.Fatal("page missing the injected fetch URL") |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | // Service secrets survive a restart: the same DB yields the same salt and |
| 218 | // signing key, so cached rows still open and signed URLs stay valid. |
| 219 | // /api/gif fronts Giphy with the platform key: keyless → 404 (clients fall |
| 220 | // back to bring-your-own-key), keyed → the upstream page passes through with |
| 221 | // every media URL rewritten to the signed /img proxy, which then serves the |
| 222 | // bytes — the browser never needs Giphy. |
| 223 | func TestFetchGifSearch(t *testing.T) { |
| 224 | ts, g := newTestFetch(t) |
| 225 | |
| 226 | // Keyless: a clean 404 so the client bridge falls through. |
| 227 | r, err := http.Get(ts.URL + "/api/gif?q=cats") |
| 228 | if err != nil { |
| 229 | t.Fatalf("gif keyless: %v", err) |
| 230 | } |
| 231 | r.Body.Close() |
| 232 | if r.StatusCode != http.StatusNotFound { |
| 233 | t.Fatalf("gif keyless: want 404, got %d", r.StatusCode) |
| 234 | } |
| 235 | |
| 236 | allowPrivatePreviewHosts = true |
| 237 | t.Cleanup(func() { allowPrivatePreviewHosts = false }) |
| 238 | |
| 239 | gifBytes := []byte("GIF89a-not-really") |
| 240 | var upstreamQuery url.Values |
| 241 | upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 242 | if r.URL.Path == "/media/cat.gif" { |
| 243 | w.Header().Set("Content-Type", "image/gif") |
| 244 | w.Write(gifBytes) |
| 245 | return |
| 246 | } |
| 247 | upstreamQuery = r.URL.Query() |
| 248 | w.Header().Set("Content-Type", "application/json") |
| 249 | json.NewEncoder(w).Encode(map[string]any{ |
| 250 | "data": []any{map[string]any{ |
| 251 | "id": "abc", "title": "Cat", |
| 252 | "images": map[string]any{ |
| 253 | "downsized": map[string]any{ |
| 254 | "url": "http://" + r.Host + "/media/cat.gif", "size": "1234", |
| 255 | }, |
| 256 | }, |
| 257 | }}, |
| 258 | }) |
| 259 | })) |
| 260 | t.Cleanup(upstream.Close) |
| 261 | oldBase := giphyAPIBase |
| 262 | giphyAPIBase = upstream.URL + "/v1/gifs/" |
| 263 | t.Cleanup(func() { giphyAPIBase = oldBase }) |
| 264 | g.giphyKey = "platform-key" |
| 265 | |
| 266 | r, err = http.Get(ts.URL + "/api/gif?q=cats&limit=5") |
| 267 | if err != nil { |
| 268 | t.Fatalf("gif search: %v", err) |
| 269 | } |
| 270 | var page struct { |
| 271 | Data []struct { |
| 272 | Images map[string]struct{ URL string `json:"url"` } `json:"images"` |
| 273 | } `json:"data"` |
| 274 | } |
| 275 | json.NewDecoder(r.Body).Decode(&page) |
| 276 | r.Body.Close() |
| 277 | if upstreamQuery.Get("api_key") != "platform-key" || upstreamQuery.Get("q") != "cats" || |
| 278 | upstreamQuery.Get("limit") != "5" { |
| 279 | t.Fatalf("upstream saw %v", upstreamQuery) |
| 280 | } |
| 281 | if len(page.Data) != 1 { |
| 282 | t.Fatalf("data = %+v", page.Data) |
| 283 | } |
| 284 | signed := page.Data[0].Images["downsized"].URL |
| 285 | if !strings.HasPrefix(signed, ts.URL+"/img?") { |
| 286 | t.Fatalf("media URL not signed onto the proxy: %q", signed) |
| 287 | } |
| 288 | ir, err := http.Get(signed) |
| 289 | if err != nil { |
| 290 | t.Fatalf("proxied gif: %v", err) |
| 291 | } |
| 292 | defer ir.Body.Close() |
| 293 | got, _ := io.ReadAll(ir.Body) |
| 294 | if ir.StatusCode != http.StatusOK || string(got) != string(gifBytes) { |
| 295 | t.Fatalf("proxied gif: status %d, bytes %q", ir.StatusCode, got) |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | func TestFetchSecretsPersist(t *testing.T) { |
| 300 | dir := t.TempDir() |
| 301 | g1, err := newFetchServer(dir + "/fetch.db") |
| 302 | if err != nil { |
| 303 | t.Fatalf("first open: %v", err) |
| 304 | } |
| 305 | salt, key := g1.salt, g1.signKey |
| 306 | g1.db.Close() |
| 307 | g2, err := newFetchServer(dir + "/fetch.db") |
| 308 | if err != nil { |
| 309 | t.Fatalf("reopen: %v", err) |
| 310 | } |
| 311 | defer g2.db.Close() |
| 312 | if g2.salt != salt || string(g2.signKey) != string(key) { |
| 313 | t.Fatal("fetch-service secrets changed across restart") |
| 314 | } |
| 315 | if len(key) != 32 || salt == "" { |
| 316 | t.Fatalf("weak secrets: key %d bytes, salt %q", len(key), salt) |
| 317 | } |
| 318 | } |
| 319 | |
| 320 | // Oversized JPEG/PNG card images shrink to card width on the way through the |
| 321 | // proxy; GIFs (animation) and already-small images pass through untouched. |
| 322 | func TestFetchImageProxyShrinks(t *testing.T) { |
| 323 | ts, g := newTestFetch(t) |
| 324 | |
| 325 | allowPrivatePreviewHosts = true |
| 326 | t.Cleanup(func() { allowPrivatePreviewHosts = false }) |
| 327 | |
| 328 | // A 2400×1200 PNG (well over fetchImgResizeOver) and a small one. |
| 329 | big := image.NewRGBA(image.Rect(0, 0, 2400, 1200)) |
| 330 | var bigBuf bytes.Buffer |
| 331 | png.Encode(&bigBuf, big) |
| 332 | var smallBuf bytes.Buffer |
| 333 | png.Encode(&smallBuf, image.NewRGBA(image.Rect(0, 0, 320, 200))) |
| 334 | |
| 335 | site := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 336 | w.Header().Set("Content-Type", "image/png") |
| 337 | if r.URL.Path == "/big.png" { |
| 338 | w.Write(bigBuf.Bytes()) |
| 339 | } else { |
| 340 | w.Write(smallBuf.Bytes()) |
| 341 | } |
| 342 | })) |
| 343 | t.Cleanup(site.Close) |
| 344 | |
| 345 | get := func(path string) []byte { |
| 346 | t.Helper() |
| 347 | req, _ := http.NewRequest("GET", ts.URL+"/api/unfurl", nil) |
| 348 | r, err := http.Get(g.proxyImageURL(&http.Request{Host: strings.TrimPrefix(ts.URL, "http://"), URL: req.URL}, site.URL+path)) |
| 349 | if err != nil { |
| 350 | t.Fatalf("proxy %s: %v", path, err) |
| 351 | } |
| 352 | defer r.Body.Close() |
| 353 | b, _ := io.ReadAll(r.Body) |
| 354 | if r.StatusCode != http.StatusOK { |
| 355 | t.Fatalf("proxy %s: status %d", path, r.StatusCode) |
| 356 | } |
| 357 | return b |
| 358 | } |
| 359 | |
| 360 | shrunk := get("/big.png") |
| 361 | cfg, _, err := image.DecodeConfig(bytes.NewReader(shrunk)) |
| 362 | if err != nil || cfg.Width != fetchImgResizeTo || cfg.Height != 600 { |
| 363 | t.Fatalf("big.png came back %dx%d err=%v (want %dx600)", cfg.Width, cfg.Height, err, fetchImgResizeTo) |
| 364 | } |
| 365 | if len(shrunk) >= bigBuf.Len() { |
| 366 | t.Fatalf("shrink didn't shrink: %d >= %d", len(shrunk), bigBuf.Len()) |
| 367 | } |
| 368 | if got := get("/small.png"); string(got) != smallBuf.String() { |
| 369 | t.Fatal("small image should pass through byte-identical") |
| 370 | } |
| 371 | } |
| 372 | |
| 373 | // The hashcash gate: unadmitted requests get a challenge, a solved challenge |
| 374 | // serves the request and mints a reusable ticket, tickets admit until expiry, |
| 375 | // and a solved challenge can't be replayed for a second ticket. |
| 376 | func TestFetchPowGate(t *testing.T) { |
| 377 | ts, g := newTestFetch(t) |
| 378 | if err := g.armPow(8); err != nil { // 8 bits: a few hundred hashes |
| 379 | t.Fatalf("armPow: %v", err) |
| 380 | } |
| 381 | |
| 382 | allowPrivatePreviewHosts = true |
| 383 | t.Cleanup(func() { allowPrivatePreviewHosts = false }) |
| 384 | site := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 385 | w.Header().Set("Content-Type", "text/html") |
| 386 | w.Write([]byte(`<head><meta property="og:title" content="Gated"></head>`)) |
| 387 | })) |
| 388 | t.Cleanup(site.Close) |
| 389 | target := "/api/unfurl?url=" + url.QueryEscape(site.URL) |
| 390 | |
| 391 | // No ticket, no pow → 401 with a challenge. |
| 392 | r, err := http.Get(ts.URL + target) |
| 393 | if err != nil { |
| 394 | t.Fatalf("gated: %v", err) |
| 395 | } |
| 396 | var gate struct { |
| 397 | Pow struct { |
| 398 | Challenge string `json:"challenge"` |
| 399 | Bits int `json:"bits"` |
| 400 | } `json:"pow"` |
| 401 | } |
| 402 | json.NewDecoder(r.Body).Decode(&gate) |
| 403 | r.Body.Close() |
| 404 | if r.StatusCode != http.StatusUnauthorized || gate.Pow.Challenge == "" || gate.Pow.Bits != 8 { |
| 405 | t.Fatalf("gate = %d %+v", r.StatusCode, gate) |
| 406 | } |
| 407 | |
| 408 | // Solve → the request itself is served AND a ticket comes back. |
| 409 | nonce := fetchSolvePow(gate.Pow.Challenge, gate.Pow.Bits) |
| 410 | r, err = http.Get(ts.URL + target + "&pow=" + url.QueryEscape(gate.Pow.Challenge) + "&nonce=" + nonce) |
| 411 | if err != nil { |
| 412 | t.Fatalf("solved: %v", err) |
| 413 | } |
| 414 | ticket := r.Header.Get("X-Fetch-Ticket") |
| 415 | var pv LinkPreview |
| 416 | json.NewDecoder(r.Body).Decode(&pv) |
| 417 | r.Body.Close() |
| 418 | if r.StatusCode != http.StatusOK || !pv.OK || pv.Title != "Gated" || ticket == "" { |
| 419 | t.Fatalf("solved: status %d pv %+v ticket %q", r.StatusCode, pv, ticket) |
| 420 | } |
| 421 | |
| 422 | // The ticket readmits; the spent challenge does not. |
| 423 | r, _ = http.Get(ts.URL + target + "&ticket=" + url.QueryEscape(ticket)) |
| 424 | r.Body.Close() |
| 425 | if r.StatusCode != http.StatusOK { |
| 426 | t.Fatalf("ticket readmit: %d", r.StatusCode) |
| 427 | } |
| 428 | r, _ = http.Get(ts.URL + target + "&pow=" + url.QueryEscape(gate.Pow.Challenge) + "&nonce=" + nonce) |
| 429 | r.Body.Close() |
| 430 | if r.StatusCode != http.StatusUnauthorized { |
| 431 | t.Fatalf("replayed challenge: want 401, got %d", r.StatusCode) |
| 432 | } |
| 433 | // Garbage tickets bounce back to the gate. |
| 434 | r, _ = http.Get(ts.URL + target + "&ticket=garbage") |
| 435 | r.Body.Close() |
| 436 | if r.StatusCode != http.StatusUnauthorized { |
| 437 | t.Fatalf("garbage ticket: want 401, got %d", r.StatusCode) |
| 438 | } |
| 439 | } |
| 440 | |
amadan