| 1 | // `lchat fetch run` — the public-web fetch service (docs/fetch.md). A shared, |
| 2 | // identity-free service that fronts *public web content* for clients: OpenGraph |
| 3 | // unfurls and an HMAC-signed image proxy for the cards' images. Like links |
| 4 | // (linkscmd.go) it's a subcommand of this binary, not a separate module — |
| 5 | // it deliberately reuses the instance's own unfurl machinery (unfurl.go: |
| 6 | // fetchOG, the SSRF dial guard, the sealed convergent cache, previewFlight). |
| 7 | // |
| 8 | // Trust model, in one breath: the fetch service sees URLs (the documented unfurl |
| 9 | // exception) attached to NOTHING — no token, no instance, no member. Behind |
| 10 | // the router it doesn't even see client IPs (nothing forwards them — that's a |
| 11 | // feature, with the honest cost that per-IP rate buckets collapse to global |
| 12 | // ones there; hashcash is the planned real gate, see docs/fetch.md). It must |
| 13 | // never grow an endpoint that touches member content: member photos/files are |
| 14 | // encrypted client-side and are categorically out of scope here. |
| 15 | // |
| 16 | // The image proxy is camo-shaped (GitHub's trick): /img serves only URLs the |
| 17 | // unfurler itself signed when it built a card, so it can't be driven as an |
| 18 | // open proxy, with no identity needed. Signing happens at response time (the |
| 19 | // cache stores the raw og:image URL), so the key could rotate without |
| 20 | // invalidating cached cards. |
| 21 | package main |
| 22 | |
| 23 | import ( |
| 24 | "bytes" |
| 25 | "crypto/hmac" |
| 26 | "crypto/rand" |
| 27 | "crypto/sha256" |
| 28 | "database/sql" |
| 29 | "encoding/base64" |
| 30 | "encoding/hex" |
| 31 | "encoding/json" |
| 32 | "flag" |
| 33 | "fmt" |
| 34 | "image" |
| 35 | "image/jpeg" |
| 36 | "image/png" |
| 37 | "io" |
| 38 | "log" |
| 39 | "net" |
| 40 | "net/http" |
| 41 | "net/url" |
| 42 | "os" |
| 43 | "strconv" |
| 44 | "strings" |
| 45 | "sync" |
| 46 | "time" |
| 47 | |
| 48 | xdraw "golang.org/x/image/draw" |
| 49 | ) |
| 50 | |
| 51 | const ( |
| 52 | fetchImgMaxBytes = 8 * 1024 * 1024 // largest card image we'll proxy |
| 53 | fetchImgTimeout = 15 * time.Second |
| 54 | fetchImgCache = "public, max-age=86400" |
| 55 | // Cards render at most a few hundred CSS px wide; og:images routinely |
| 56 | // arrive at several thousand. Anything wider than Over re-encodes at To. |
| 57 | fetchImgResizeOver = 1600 |
| 58 | fetchImgResizeTo = 1200 |
| 59 | ) |
| 60 | |
| 61 | // fetchImgClient fetches card images. Same SSRF posture as ogClient — every |
| 62 | // dial (and every redirect hop) re-validates the resolved IP. |
| 63 | var fetchImgClient = &http.Client{ |
| 64 | Timeout: fetchImgTimeout, |
| 65 | Transport: &http.Transport{ |
| 66 | DialContext: (&net.Dialer{ |
| 67 | Timeout: fetchImgTimeout, |
| 68 | Control: guardDialControl, |
| 69 | }).DialContext, |
| 70 | TLSHandshakeTimeout: fetchImgTimeout, |
| 71 | ResponseHeaderTimeout: fetchImgTimeout, |
| 72 | DisableKeepAlives: true, |
| 73 | }, |
| 74 | CheckRedirect: func(req *http.Request, via []*http.Request) error { |
| 75 | if len(via) >= 5 { |
| 76 | return fmt.Errorf("too many redirects") |
| 77 | } |
| 78 | if req.URL.Scheme != "http" && req.URL.Scheme != "https" { |
| 79 | return fmt.Errorf("non-http redirect") |
| 80 | } |
| 81 | return nil |
| 82 | }, |
| 83 | } |
| 84 | |
| 85 | type fetchServer struct { |
| 86 | db *sql.DB |
| 87 | salt string // service-wide preview salt (convergent-encryption pepper) |
| 88 | signKey []byte // HMAC key for /img URLs; minted once, lives only in the DB |
| 89 | // inflight coalesces concurrent unfurls exactly like the instance's |
| 90 | // (main.go): one fetch per URL+language, everyone shares the result. |
| 91 | inflight sync.Map |
| 92 | // Ceilings, not budgets — and behind the router every caller shares one |
| 93 | // bucket (no client IPs are forwarded, on purpose). See the package doc. |
| 94 | unfurlLimiter *modLimiter |
| 95 | imgLimiter *modLimiter |
| 96 | gifLimiter *modLimiter |
| 97 | // The platform's Giphy API key (LCHAT_GIPHY_KEY). Empty disables /api/gif |
| 98 | // (404), and clients fall back to the space's bring-your-own-key path. |
| 99 | giphyKey string |
| 100 | // Hashcash admission (fetchpow.go); nil = off (the default). Its seal key |
| 101 | // is minted into the DB alongside the salt so restarts keep tickets valid. |
| 102 | pow *fetchPowGate |
| 103 | powSealKey []byte |
| 104 | } |
| 105 | |
| 106 | func newFetchServer(dbPath string) (*fetchServer, error) { |
| 107 | d, err := sql.Open("sqlite", dbPath) |
| 108 | if err != nil { |
| 109 | return nil, err |
| 110 | } |
| 111 | d.SetMaxOpenConns(1) |
| 112 | for _, p := range []string{ |
| 113 | "PRAGMA journal_mode=WAL", |
| 114 | "PRAGMA busy_timeout=5000", |
| 115 | "PRAGMA synchronous=NORMAL", |
| 116 | } { |
| 117 | if _, err := d.Exec(p); err != nil { |
| 118 | return nil, err |
| 119 | } |
| 120 | } |
| 121 | for _, ddl := range []string{ |
| 122 | `CREATE TABLE IF NOT EXISTS config(key TEXT PRIMARY KEY, value TEXT NOT NULL)`, |
| 123 | // Same sealed shape as the instance's link_previews_sealed: the row key |
| 124 | // is a salted hash and the blob opens only with a key derived from the |
| 125 | // language+URL, so the shared cache at rest reveals nothing a reader |
| 126 | // doesn't already know the URL for. |
| 127 | `CREATE TABLE IF NOT EXISTS fetch_previews( |
| 128 | url_hash TEXT PRIMARY KEY, |
| 129 | blob TEXT NOT NULL, |
| 130 | fetched_at INTEGER NOT NULL)`, |
| 131 | } { |
| 132 | if _, err := d.Exec(ddl); err != nil { |
| 133 | return nil, err |
| 134 | } |
| 135 | } |
| 136 | g := &fetchServer{ |
| 137 | db: d, |
| 138 | unfurlLimiter: newModLimiter(30, 30.0/60), |
| 139 | gifLimiter: newModLimiter(60, 60.0/60), |
| 140 | imgLimiter: newModLimiter(240, 120.0/60), |
| 141 | } |
| 142 | if g.salt, err = fetchConfigSecret(d, "preview_salt"); err != nil { |
| 143 | return nil, err |
| 144 | } |
| 145 | keyHex, err := fetchConfigSecret(d, "img_sign_key") |
| 146 | if err != nil { |
| 147 | return nil, err |
| 148 | } |
| 149 | if g.signKey, err = hex.DecodeString(keyHex); err != nil { |
| 150 | return nil, err |
| 151 | } |
| 152 | powHex, err := fetchConfigSecret(d, "pow_seal_key") |
| 153 | if err != nil { |
| 154 | return nil, err |
| 155 | } |
| 156 | if g.powSealKey, err = hex.DecodeString(powHex); err != nil { |
| 157 | return nil, err |
| 158 | } |
| 159 | return g, nil |
| 160 | } |
| 161 | |
| 162 | // armPow turns the hashcash gate on at the given difficulty (0 = off). |
| 163 | func (g *fetchServer) armPow(bits int) error { |
| 164 | gate, err := newFetchPowGate(bits, g.powSealKey) |
| 165 | if err != nil { |
| 166 | return err |
| 167 | } |
| 168 | g.pow = gate |
| 169 | return nil |
| 170 | } |
| 171 | |
| 172 | // fetchConfigSecret returns the named config value, minting a random one on |
| 173 | // first boot — the same insert-if-absent pattern as the instance's |
| 174 | // previewSalt, so restarts and concurrent starts converge on one value. |
| 175 | func fetchConfigSecret(d *sql.DB, key string) (string, error) { |
| 176 | buf := make([]byte, 32) |
| 177 | if _, err := rand.Read(buf); err != nil { |
| 178 | return "", err |
| 179 | } |
| 180 | if _, err := d.Exec( |
| 181 | "INSERT OR IGNORE INTO config(key, value) VALUES(?, ?)", key, hex.EncodeToString(buf)); err != nil { |
| 182 | return "", err |
| 183 | } |
| 184 | var v string |
| 185 | err := d.QueryRow("SELECT value FROM config WHERE key = ?", key).Scan(&v) |
| 186 | return v, err |
| 187 | } |
| 188 | |
| 189 | func (g *fetchServer) handler() http.Handler { |
| 190 | mux := http.NewServeMux() |
| 191 | // The two open endpoints sit behind the (optional) hashcash gate; /img |
| 192 | // needs none — its HMAC signature is already the admission. |
| 193 | mux.HandleFunc("GET /api/unfurl", g.powGated(g.serveUnfurl)) |
| 194 | mux.HandleFunc("GET /img", g.serveImage) |
| 195 | mux.HandleFunc("GET /api/gif", g.powGated(g.serveGifSearch)) |
| 196 | mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { |
| 197 | fmt.Fprintln(w, "ok") |
| 198 | }) |
| 199 | mux.HandleFunc("GET /api/version", func(w http.ResponseWriter, r *http.Request) { |
| 200 | writeJSON(w, http.StatusOK, map[string]any{"service": "fetch", "build": buildVersion}) |
| 201 | }) |
| 202 | // The fetch service has no pages of its own; a stray visit goes to the real site. |
| 203 | mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { |
| 204 | http.Redirect(w, r, "https://elevenmessenger.com", http.StatusFound) |
| 205 | }) |
| 206 | // Every caller is a browser on some OTHER origin (each space is its own |
| 207 | // origin; the service serves them all), so without CORS headers fetch() |
| 208 | // can never read a response and every client silently falls back to its |
| 209 | // instance path. `*` is the honest scope: the service is anonymous by |
| 210 | // design — no cookies, no credentials, nothing origin-scoped to protect. |
| 211 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 212 | w.Header().Set("Access-Control-Allow-Origin", "*") |
| 213 | // The pow gate hands fresh tickets back on a response header; without |
| 214 | // this, cross-origin JS can see the response but not that header. |
| 215 | w.Header().Set("Access-Control-Expose-Headers", "X-Fetch-Ticket") |
| 216 | mux.ServeHTTP(w, r) |
| 217 | }) |
| 218 | } |
| 219 | |
| 220 | // powGated wraps an open endpoint in the hashcash gate (nil gate = pass). |
| 221 | // Admission rides query params, not headers — a custom request header would |
| 222 | // cost every cross-origin caller a CORS preflight round trip. |
| 223 | // |
| 224 | // - ?ticket= a sealed, unexpired ticket → in, rate-bucketed by the ticket |
| 225 | // - ?pow=&nonce= a solved challenge → consumed, request served, and a |
| 226 | // fresh ticket returned in X-Fetch-Ticket for the requests that follow |
| 227 | // - neither → 401 {"pow":{"challenge","bits"}} — the mint rides the |
| 228 | // rejection, so admission costs one round trip, not two |
| 229 | func (g *fetchServer) powGated(next http.HandlerFunc) http.HandlerFunc { |
| 230 | return func(w http.ResponseWriter, r *http.Request) { |
| 231 | if g.pow == nil { |
| 232 | next(w, r) |
| 233 | return |
| 234 | } |
| 235 | q := r.URL.Query() |
| 236 | if t := q.Get("ticket"); t != "" && g.pow.ticketOK(t) { |
| 237 | next(w, r) |
| 238 | return |
| 239 | } |
| 240 | if c := q.Get("pow"); c != "" && g.pow.verifyAndConsume(c, q.Get("nonce")) == nil { |
| 241 | if t, err := g.pow.mintTicket(); err == nil { |
| 242 | w.Header().Set("X-Fetch-Ticket", t) |
| 243 | } |
| 244 | next(w, r) |
| 245 | return |
| 246 | } |
| 247 | ch, err := g.pow.mintChallenge() |
| 248 | if err != nil { |
| 249 | httpError(w, http.StatusInternalServerError, "pow mint failed") |
| 250 | return |
| 251 | } |
| 252 | writeJSON(w, http.StatusUnauthorized, map[string]any{ |
| 253 | "pow": map[string]any{"challenge": ch, "bits": g.pow.bits}, |
| 254 | }) |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | // rateKey buckets a gated request by its ticket (each solved client gets its |
| 259 | // own bucket); ungated requests keep the caller-IP key, with its documented |
| 260 | // behind-the-router collapse. |
| 261 | func (g *fetchServer) rateKey(r *http.Request) string { |
| 262 | if g.pow != nil { |
| 263 | if t := r.URL.Query().Get("ticket"); t != "" { |
| 264 | return "t:" + t |
| 265 | } |
| 266 | } |
| 267 | return fetchCallerKey(r) |
| 268 | } |
| 269 | |
| 270 | // fetchCallerKey is the rate-limit bucket key. Mirrors linkscmd.go — behind |
| 271 | // the router (unix socket, no forwarded IPs) this degrades to one shared |
| 272 | // bucket, which is the accepted posture for now (package doc). |
| 273 | func fetchCallerKey(r *http.Request) string { |
| 274 | ip, _, err := net.SplitHostPort(r.RemoteAddr) |
| 275 | if err != nil { |
| 276 | return r.RemoteAddr |
| 277 | } |
| 278 | return ip |
| 279 | } |
| 280 | |
| 281 | // serveUnfurl is the instance handler's anonymous twin: no token, same URL |
| 282 | // validation, same language handling, same stale-serve semantics — but the |
| 283 | // card's image URL is rewritten to the signed /img proxy before it leaves. |
| 284 | func (g *fetchServer) serveUnfurl(w http.ResponseWriter, r *http.Request) { |
| 285 | if !g.unfurlLimiter.allow(g.rateKey(r)) { |
| 286 | httpError(w, http.StatusTooManyRequests, "slow down") |
| 287 | return |
| 288 | } |
| 289 | raw := strings.TrimSpace(r.URL.Query().Get("url")) |
| 290 | u, err := url.Parse(raw) |
| 291 | if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || len(raw) > 2048 { |
| 292 | httpError(w, http.StatusBadRequest, "bad url") |
| 293 | return |
| 294 | } |
| 295 | norm := u.String() |
| 296 | lang := previewLang(r.Header.Get("Accept-Language")) |
| 297 | |
| 298 | if pv, ok := g.getPreview(norm, lang); ok { |
| 299 | writeJSON(w, http.StatusOK, g.signedCard(r, pv)) |
| 300 | ttl := int64(previewFreshTTL) |
| 301 | if !pv.OK { |
| 302 | ttl = previewFailTTL |
| 303 | } |
| 304 | if time.Now().Unix()-pv.FetchedAt > ttl { |
| 305 | go g.fetchAndCache(norm, lang) |
| 306 | } |
| 307 | return |
| 308 | } |
| 309 | writeJSON(w, http.StatusOK, g.signedCard(r, g.fetchAndCache(norm, lang))) |
| 310 | } |
| 311 | |
| 312 | // fetchAndCache mirrors the instance's fetchAndCachePreview (main.go), |
| 313 | // including the mid-fetch waiters fix — against the fetch service's own store. |
| 314 | func (g *fetchServer) fetchAndCache(norm, lang string) LinkPreview { |
| 315 | flightKey := previewCacheKey(lang, norm) |
| 316 | f := &previewFlight{done: make(chan struct{})} |
| 317 | if cur, busy := g.inflight.LoadOrStore(flightKey, f); busy { |
| 318 | flight := cur.(*previewFlight) |
| 319 | <-flight.done |
| 320 | return flight.pv |
| 321 | } |
| 322 | defer func() { |
| 323 | g.inflight.Delete(flightKey) |
| 324 | close(f.done) |
| 325 | }() |
| 326 | |
| 327 | pv, err := fetchOG(norm, lang) |
| 328 | if err != nil { |
| 329 | pv = LinkPreview{URL: norm, OK: false} |
| 330 | } |
| 331 | pv.URL = norm |
| 332 | pv.FetchedAt = time.Now().Unix() |
| 333 | g.putPreview(pv, lang) |
| 334 | f.pv = pv |
| 335 | return pv |
| 336 | } |
| 337 | |
| 338 | func (g *fetchServer) getPreview(u, lang string) (LinkPreview, bool) { |
| 339 | key := previewCacheKey(lang, u) |
| 340 | var blob string |
| 341 | var fetchedAt int64 |
| 342 | err := g.db.QueryRow( |
| 343 | "SELECT blob, fetched_at FROM fetch_previews WHERE url_hash = ?", previewURLHash(g.salt, key), |
| 344 | ).Scan(&blob, &fetchedAt) |
| 345 | if err != nil { |
| 346 | return LinkPreview{}, false |
| 347 | } |
| 348 | pv, ok := openPreview(g.salt, key, blob) |
| 349 | if !ok { |
| 350 | return LinkPreview{}, false |
| 351 | } |
| 352 | pv.URL = u |
| 353 | pv.FetchedAt = fetchedAt |
| 354 | return pv, true |
| 355 | } |
| 356 | |
| 357 | func (g *fetchServer) putPreview(pv LinkPreview, lang string) { |
| 358 | key := previewCacheKey(lang, pv.URL) |
| 359 | blob, err := sealPreview(g.salt, key, pv) |
| 360 | if err != nil { |
| 361 | return |
| 362 | } |
| 363 | // Same put-time TTL sweep as the instance cache: prunes with activity. |
| 364 | g.db.Exec("DELETE FROM fetch_previews WHERE fetched_at < ?", time.Now().Unix()-previewMaxAge) |
| 365 | g.db.Exec( |
| 366 | `INSERT INTO fetch_previews(url_hash, blob, fetched_at) VALUES(?, ?, ?) |
| 367 | ON CONFLICT(url_hash) DO UPDATE SET blob=excluded.blob, fetched_at=excluded.fetched_at`, |
| 368 | previewURLHash(g.salt, key), blob, pv.FetchedAt) |
| 369 | } |
| 370 | |
| 371 | // signedCard swaps a card's raw og:image URL for the signed /img proxy URL, |
| 372 | // so the viewer's browser talks to the fetch service, not the image's host (no member |
| 373 | // IP reaches it, and http-only images stop dying as mixed content). |
| 374 | func (g *fetchServer) signedCard(r *http.Request, pv LinkPreview) LinkPreview { |
| 375 | if pv.Image != "" { |
| 376 | pv.Image = g.proxyImageURL(r, pv.Image) |
| 377 | } |
| 378 | return pv |
| 379 | } |
| 380 | |
| 381 | func (g *fetchServer) proxyImageURL(r *http.Request, img string) string { |
| 382 | mac := hmac.New(sha256.New, g.signKey) |
| 383 | mac.Write([]byte(img)) |
| 384 | scheme := "https" |
| 385 | if r.TLS == nil && fetchLoopbackHost(r.Host) { |
| 386 | scheme = "http" // tests and bare local runs; behind the router TLS is the router's |
| 387 | } |
| 388 | return scheme + "://" + r.Host + "/img?u=" + |
| 389 | base64.RawURLEncoding.EncodeToString([]byte(img)) + |
| 390 | "&s=" + hex.EncodeToString(mac.Sum(nil)) |
| 391 | } |
| 392 | |
| 393 | func fetchLoopbackHost(hostport string) bool { |
| 394 | host, _, err := net.SplitHostPort(hostport) |
| 395 | if err != nil { |
| 396 | host = hostport |
| 397 | } |
| 398 | if host == "localhost" { |
| 399 | return true |
| 400 | } |
| 401 | ip := net.ParseIP(host) |
| 402 | return ip != nil && ip.IsLoopback() |
| 403 | } |
| 404 | |
| 405 | // serveImage is the camo half: verify the signature (only URLs this service |
| 406 | // itself put in a card are proxyable — never an open proxy), then fetch and |
| 407 | // re-serve the image over our origin. Buffered rather than streamed so an |
| 408 | // oversize image is rejected cleanly instead of truncated mid-body; the size |
| 409 | // cap bounds the memory. |
| 410 | func (g *fetchServer) serveImage(w http.ResponseWriter, r *http.Request) { |
| 411 | if !g.imgLimiter.allow(fetchCallerKey(r)) { |
| 412 | httpError(w, http.StatusTooManyRequests, "slow down") |
| 413 | return |
| 414 | } |
| 415 | rawImg, err := base64.RawURLEncoding.DecodeString(r.URL.Query().Get("u")) |
| 416 | if err != nil { |
| 417 | httpError(w, http.StatusBadRequest, "bad url") |
| 418 | return |
| 419 | } |
| 420 | sig, err := hex.DecodeString(r.URL.Query().Get("s")) |
| 421 | if err != nil { |
| 422 | httpError(w, http.StatusBadRequest, "bad signature") |
| 423 | return |
| 424 | } |
| 425 | mac := hmac.New(sha256.New, g.signKey) |
| 426 | mac.Write(rawImg) |
| 427 | if !hmac.Equal(sig, mac.Sum(nil)) { |
| 428 | httpError(w, http.StatusForbidden, "bad signature") |
| 429 | return |
| 430 | } |
| 431 | imgURL, err := url.Parse(string(rawImg)) |
| 432 | if err != nil || (imgURL.Scheme != "http" && imgURL.Scheme != "https") { |
| 433 | httpError(w, http.StatusBadRequest, "bad url") |
| 434 | return |
| 435 | } |
| 436 | |
| 437 | req, err := http.NewRequestWithContext(r.Context(), "GET", imgURL.String(), nil) |
| 438 | if err != nil { |
| 439 | httpError(w, http.StatusBadGateway, "fetch failed") |
| 440 | return |
| 441 | } |
| 442 | req.Header.Set("User-Agent", "lchat-fetch/1.0 (+https://elevenmessenger.com)") |
| 443 | req.Header.Set("Accept", "image/*") |
| 444 | resp, err := fetchImgClient.Do(req) |
| 445 | if err != nil { |
| 446 | httpError(w, http.StatusBadGateway, "fetch failed") |
| 447 | return |
| 448 | } |
| 449 | defer resp.Body.Close() |
| 450 | ct := strings.ToLower(resp.Header.Get("Content-Type")) |
| 451 | if resp.StatusCode != http.StatusOK || !strings.HasPrefix(ct, "image/") { |
| 452 | httpError(w, http.StatusBadGateway, "not an image") |
| 453 | return |
| 454 | } |
| 455 | body, err := io.ReadAll(io.LimitReader(resp.Body, fetchImgMaxBytes+1)) |
| 456 | if err != nil || len(body) > fetchImgMaxBytes { |
| 457 | httpError(w, http.StatusBadGateway, "image too large") |
| 458 | return |
| 459 | } |
| 460 | body = shrinkCardImage(ct, body) |
| 461 | w.Header().Set("Content-Type", resp.Header.Get("Content-Type")) |
| 462 | w.Header().Set("Cache-Control", fetchImgCache) |
| 463 | w.Header().Set("X-Content-Type-Options", "nosniff") |
| 464 | w.Write(body) |
| 465 | } |
| 466 | |
| 467 | // shrinkCardImage re-encodes an oversized JPEG/PNG down to card width — an |
| 468 | // og:image is often thousands of px wide for a card that renders at a few |
| 469 | // hundred, and every viewer pays that transfer. Only those two formats: a |
| 470 | // resized GIF would freeze its animation, and nothing else here has a stdlib |
| 471 | // encoder. Every failure path returns the original bytes — polish must never |
| 472 | // break a card — and a "shrink" that came out bigger keeps the original too. |
| 473 | func shrinkCardImage(ct string, body []byte) []byte { |
| 474 | if !strings.HasPrefix(ct, "image/jpeg") && !strings.HasPrefix(ct, "image/png") { |
| 475 | return body |
| 476 | } |
| 477 | cfg, kind, err := image.DecodeConfig(bytes.NewReader(body)) |
| 478 | if err != nil || cfg.Width <= fetchImgResizeOver { |
| 479 | return body |
| 480 | } |
| 481 | // Decompression bomb: a tiny file can claim enormous pixel dims. Don't |
| 482 | // decode those ourselves — pass the bytes through and let the browser's |
| 483 | // own limits deal with it. |
| 484 | if cfg.Height <= 0 || cfg.Width*cfg.Height > 40<<20 { |
| 485 | return body |
| 486 | } |
| 487 | src, _, err := image.Decode(bytes.NewReader(body)) |
| 488 | if err != nil { |
| 489 | return body |
| 490 | } |
| 491 | h := cfg.Height * fetchImgResizeTo / cfg.Width |
| 492 | if h < 1 { |
| 493 | h = 1 |
| 494 | } |
| 495 | dst := image.NewRGBA(image.Rect(0, 0, fetchImgResizeTo, h)) |
| 496 | xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), xdraw.Over, nil) |
| 497 | var out bytes.Buffer |
| 498 | switch kind { |
| 499 | case "png": |
| 500 | err = png.Encode(&out, dst) |
| 501 | default: |
| 502 | err = jpeg.Encode(&out, dst, &jpeg.Options{Quality: 80}) |
| 503 | } |
| 504 | if err != nil || out.Len() >= len(body) { |
| 505 | return body |
| 506 | } |
| 507 | return out.Bytes() |
| 508 | } |
| 509 | |
| 510 | // giphyAPIBase is a var so tests can stand a fake upstream in. |
| 511 | var giphyAPIBase = "https://api.giphy.com/v1/gifs/" |
| 512 | |
| 513 | // serveGifSearch fronts Giphy search/trending with the PLATFORM's API key |
| 514 | // (docs/fetch.md § Giphy), so a space needs no key of its own. The response is |
| 515 | // Giphy's own page shape, passed through untouched except that every giphy- |
| 516 | // hosted media URL is rewritten to this service's signed /img proxy — the |
| 517 | // browser then never talks to Giphy at all: search terms reach it from this |
| 518 | // service's IP (identity-free), and GIF bytes come back over our origin. |
| 519 | // No key configured → 404, and the client falls back to the space's |
| 520 | // bring-your-own-key path (hostBridge.gif in shell.js). |
| 521 | func (g *fetchServer) serveGifSearch(w http.ResponseWriter, r *http.Request) { |
| 522 | if g.giphyKey == "" { |
| 523 | httpError(w, http.StatusNotFound, "no gif upstream") |
| 524 | return |
| 525 | } |
| 526 | if !g.gifLimiter.allow(g.rateKey(r)) { |
| 527 | httpError(w, http.StatusTooManyRequests, "slow down") |
| 528 | return |
| 529 | } |
| 530 | q := strings.TrimSpace(r.URL.Query().Get("q")) |
| 531 | if runes := []rune(q); len(runes) > 100 { |
| 532 | q = string(runes[:100]) |
| 533 | } |
| 534 | limit := 24 |
| 535 | if n, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil && n >= 1 && n <= 50 { |
| 536 | limit = n |
| 537 | } |
| 538 | up := giphyAPIBase + "trending?api_key=" + url.QueryEscape(g.giphyKey) + |
| 539 | "&limit=" + strconv.Itoa(limit) + "&rating=pg-13" |
| 540 | if q != "" { |
| 541 | up = giphyAPIBase + "search?api_key=" + url.QueryEscape(g.giphyKey) + |
| 542 | "&q=" + url.QueryEscape(q) + "&limit=" + strconv.Itoa(limit) + "&rating=pg-13" |
| 543 | } |
| 544 | req, err := http.NewRequestWithContext(r.Context(), "GET", up, nil) |
| 545 | if err != nil { |
| 546 | httpError(w, http.StatusBadGateway, "gif search failed") |
| 547 | return |
| 548 | } |
| 549 | resp, err := fetchImgClient.Do(req) |
| 550 | if err != nil { |
| 551 | httpError(w, http.StatusBadGateway, "gif search failed") |
| 552 | return |
| 553 | } |
| 554 | defer resp.Body.Close() |
| 555 | if resp.StatusCode != http.StatusOK { |
| 556 | httpError(w, http.StatusBadGateway, "gif search failed") |
| 557 | return |
| 558 | } |
| 559 | var page any |
| 560 | if json.NewDecoder(io.LimitReader(resp.Body, 4*1024*1024)).Decode(&page) != nil { |
| 561 | httpError(w, http.StatusBadGateway, "gif search failed") |
| 562 | return |
| 563 | } |
| 564 | // Results are transient search pages — never cache them (the media URLs |
| 565 | // inside are signed against this service's key anyway). |
| 566 | w.Header().Set("Cache-Control", "no-store") |
| 567 | writeJSON(w, http.StatusOK, g.signGiphyMedia(r, page)) |
| 568 | } |
| 569 | |
| 570 | // signGiphyMedia walks a decoded Giphy page and swaps every giphy-hosted URL |
| 571 | // string for this service's signed /img URL, wherever it sits — Giphy nests |
| 572 | // renditions several levels deep and adds new ones over time, so a generic |
| 573 | // walk beats a field list. Only giphy hosts are signed: other strings (ids, |
| 574 | // titles, third-party "source" pages we'd never want proxyable) pass through. |
| 575 | func (g *fetchServer) signGiphyMedia(r *http.Request, v any) any { |
| 576 | switch t := v.(type) { |
| 577 | case map[string]any: |
| 578 | for k, vv := range t { |
| 579 | t[k] = g.signGiphyMedia(r, vv) |
| 580 | } |
| 581 | return t |
| 582 | case []any: |
| 583 | for i, vv := range t { |
| 584 | t[i] = g.signGiphyMedia(r, vv) |
| 585 | } |
| 586 | return t |
| 587 | case string: |
| 588 | ok := func(s string) bool { return s == "https" || (s == "http" && allowPrivatePreviewHosts) } |
| 589 | if u, err := url.Parse(t); err == nil && ok(u.Scheme) && giphyMediaHost(u.Hostname()) { |
| 590 | return g.proxyImageURL(r, t) |
| 591 | } |
| 592 | return t |
| 593 | default: |
| 594 | return t |
| 595 | } |
| 596 | } |
| 597 | |
| 598 | func giphyMediaHost(h string) bool { |
| 599 | if allowPrivatePreviewHosts { |
| 600 | return true // tests: the fake upstream's media lives on loopback |
| 601 | } |
| 602 | h = strings.ToLower(h) |
| 603 | return h == "giphy.com" || strings.HasSuffix(h, ".giphy.com") |
| 604 | } |
| 605 | |
| 606 | // runFetchCmd dispatches `lchat fetch <command>`, mirroring runLinksCmd. |
| 607 | func runFetchCmd(args []string) { |
| 608 | if len(args) < 1 { |
| 609 | fmt.Fprintln(os.Stderr, `usage: lchat fetch <command> |
| 610 | |
| 611 | commands: |
| 612 | run serve the public-web fetch service (unfurl + signed image proxy; docs/fetch.md)`) |
| 613 | os.Exit(2) |
| 614 | } |
| 615 | switch args[0] { |
| 616 | case "run": |
| 617 | runFetchRun(args[1:]) |
| 618 | default: |
| 619 | die("fetch: unknown command %q", args[0]) |
| 620 | } |
| 621 | } |
| 622 | |
| 623 | // runFetchRun is `lchat fetch run`. |
| 624 | func runFetchRun(args []string) { |
| 625 | fs := flag.NewFlagSet("fetch", flag.ExitOnError) |
| 626 | addr := fs.String("addr", envOr("ADDR", ":1749"), "listen address (env: ADDR)") |
| 627 | socket := fs.String("socket", envOr("LCHAT_SOCKET", ""), "unix socket to serve on instead of -addr (env: LCHAT_SOCKET) — how the router fronts this service") |
| 628 | dbPath := fs.String("db", envOr("LCHAT_FETCH_DB", "fetch.db"), "sqlite path for the sealed preview cache + service secrets (env: LCHAT_FETCH_DB)") |
| 629 | _ = fs.Parse(args) |
| 630 | |
| 631 | g, err := newFetchServer(*dbPath) |
| 632 | if err != nil { |
| 633 | die("fetch: %v", err) |
| 634 | } |
| 635 | // The one env secret: the platform Giphy key (unit env file /data/fetch.env). |
| 636 | // Absent, /api/gif 404s and clients use the space's own key instead. |
| 637 | g.giphyKey = strings.TrimSpace(os.Getenv("LCHAT_GIPHY_KEY")) |
| 638 | // Hashcash difficulty (LCHAT_FETCH_POW_BITS, 0/unset = off): arm before |
| 639 | // pointing the hosted fleet here; leave off for dev and self-hosters. |
| 640 | if bits, _ := strconv.Atoi(strings.TrimSpace(os.Getenv("LCHAT_FETCH_POW_BITS"))); bits > 0 { |
| 641 | if err := g.armPow(bits); err != nil { |
| 642 | die("fetch: pow: %v", err) |
| 643 | } |
| 644 | log.Printf("fetch: hashcash admission armed (%d bits)", bits) |
| 645 | } |
| 646 | ln, addrDesc, err := listen(*addr, *socket) |
| 647 | if err != nil { |
| 648 | die("fetch: %v", err) |
| 649 | } |
| 650 | // Deliberately the only startup log line — the fetch service never logs URLs. |
| 651 | log.Printf("fetch: serving on %s", addrDesc) |
| 652 | srv := &http.Server{Handler: g.handler(), ReadHeaderTimeout: 10 * time.Second} |
| 653 | if err := srv.Serve(ln); err != nil { |
| 654 | log.Fatal(err) |
| 655 | } |
| 656 | } |
| 657 | |
amadan