amadan

eleven / messenger Public

server: OpenGraph link previews (SSRF-guarded, cached)

GET /api/unfurl?token=&url= fetches a posted URL, parses og:/twitter:/<title>, and caches it in link_previews. Serves cached results immediately and refreshes stale ones in the background; single-flight per URL; failures cached briefly so they retry sooner.

SSRF guard: a custom dialer rejects loopback/private/link-local/CGNAT/unspecified IPs on every connection (defeats DNS rebinding), with size/redirect/timeout caps and an html-only content check. allowPrivatePreviewHosts opens loopback for tests only.

This is the one place the server sees plaintext URLs (message text stays encrypted); easy to gate behind a setting later.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Paul Campbell 132188955ff7e61b5b6b1b6f53dd4549e36be08b parent b625d1e
4 files changed, +417 −0
  • db.go +42 −0
  • main.go +61 −0
  • unfurl.go +213 −0
  • unfurl_test.go +101 −0
diff --git a/db.go b/db.go
index d76fbeb..1535268 100644
--- a/db.go
+++ b/db.go
@@ -96,10 +96,52 @@ CREATE TABLE IF NOT EXISTS blobs (
data BLOB NOT NULL,
size INTEGER NOT NULL,
created_at INTEGER NOT NULL
+);
+CREATE TABLE IF NOT EXISTS link_previews (
+ url TEXT PRIMARY KEY,
+ title TEXT NOT NULL DEFAULT '',
+ description TEXT NOT NULL DEFAULT '',
+ image TEXT NOT NULL DEFAULT '',
+ ok INTEGER NOT NULL DEFAULT 0,
+ fetched_at INTEGER NOT NULL
);`)
return err
}
+// ---- link previews (cache) ----------------------------------------------
+
+func (db *DB) getLinkPreview(u string) (LinkPreview, bool, error) {
+ var pv LinkPreview
+ var ok int
+ err := db.sql.QueryRow(
+ "SELECT url, title, description, image, ok, fetched_at FROM link_previews WHERE url = ?", u,
+ ).Scan(&pv.URL, &pv.Title, &pv.Description, &pv.Image, &ok, &pv.FetchedAt)
+ if err == sql.ErrNoRows {
+ return LinkPreview{}, false, nil
+ }
+ if err != nil {
+ return LinkPreview{}, false, err
+ }
+ pv.OK = ok != 0
+ return pv, true, nil
+}
+
+func (db *DB) putLinkPreview(pv LinkPreview) error {
+ ok := 0
+ if pv.OK {
+ ok = 1
+ }
+ _, err := db.sql.Exec(
+ `INSERT INTO link_previews(url, title, description, image, ok, fetched_at)
+ VALUES(?, ?, ?, ?, ?, ?)
+ ON CONFLICT(url) DO UPDATE SET
+ title=excluded.title, description=excluded.description,
+ image=excluded.image, ok=excluded.ok, fetched_at=excluded.fetched_at`,
+ pv.URL, pv.Title, pv.Description, pv.Image, ok, pv.FetchedAt,
+ )
+ return err
+}
+
// ---- config -------------------------------------------------------------
func (db *DB) getConfig(key string) (string, bool, error) {
diff --git a/main.go b/main.go
index 8d3869e..a562995 100644
--- a/main.go
+++ b/main.go
@@ -9,9 +9,12 @@ import (
"io/fs"
"log"
"net/http"
+ "net/url"
"os"
"strconv"
"strings"
+ "sync"
+ "time"
)
// maxUploadBytes caps an encrypted attachment upload (10 MB of ciphertext).
@@ -26,6 +29,8 @@ type Server struct {
pages map[string][]byte
vapidPub string
vapidPriv string
+ // inflight de-duplicates concurrent unfurls of the same URL.
+ inflight sync.Map
}
func main() {
@@ -101,6 +106,10 @@ func (s *Server) handler() http.Handler {
mux.HandleFunc("POST /api/upload", s.uploadBlob)
mux.HandleFunc("GET /api/blob/{id}", s.getBlobHandler)
+ // Link previews: the server fetches a posted URL to build an OpenGraph card,
+ // caches it, and serves cached results immediately (refreshing stale ones).
+ mux.HandleFunc("GET /api/unfurl", s.unfurl)
+
// Invite lifecycle (used by the join page)
mux.HandleFunc("GET /api/invite/{token}", s.getInvite)
mux.HandleFunc("POST /api/invite/{token}/claim", s.claimInvite)
@@ -240,6 +249,58 @@ func (s *Server) getBlobHandler(w http.ResponseWriter, r *http.Request) {
w.Write(data)
}
+// ---- link previews ------------------------------------------------------
+
+func (s *Server) unfurl(w http.ResponseWriter, r *http.Request) {
+ if !s.tokenOK(r.URL.Query().Get("token")) {
+ httpError(w, http.StatusForbidden, "invalid invite")
+ return
+ }
+ raw := strings.TrimSpace(r.URL.Query().Get("url"))
+ u, err := url.Parse(raw)
+ if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || len(raw) > 2048 {
+ httpError(w, http.StatusBadRequest, "bad url")
+ return
+ }
+ norm := u.String()
+
+ // Cache hit: serve it now; if it's stale, refresh in the background.
+ if pv, ok, _ := s.db.getLinkPreview(norm); ok {
+ writeJSON(w, http.StatusOK, pv)
+ ttl := int64(previewFreshTTL)
+ if !pv.OK {
+ ttl = previewFailTTL
+ }
+ if time.Now().Unix()-pv.FetchedAt > ttl {
+ go s.fetchAndCachePreview(norm)
+ }
+ return
+ }
+ // Miss: fetch synchronously (de-duped), cache, return.
+ writeJSON(w, http.StatusOK, s.fetchAndCachePreview(norm))
+}
+
+// fetchAndCachePreview fetches+stores a preview, ensuring only one fetch per URL
+// runs at a time. Concurrent callers get whatever is currently cached.
+func (s *Server) fetchAndCachePreview(norm string) LinkPreview {
+ if _, busy := s.inflight.LoadOrStore(norm, true); busy {
+ if pv, ok, _ := s.db.getLinkPreview(norm); ok {
+ return pv
+ }
+ return LinkPreview{URL: norm}
+ }
+ defer s.inflight.Delete(norm)
+
+ pv, err := fetchOG(norm)
+ if err != nil {
+ pv = LinkPreview{URL: norm, OK: false}
+ }
+ pv.URL = norm
+ pv.FetchedAt = time.Now().Unix()
+ _ = s.db.putLinkPreview(pv)
+ return pv
+}
+
// ---- invite (join flow) -------------------------------------------------
func (s *Server) getInvite(w http.ResponseWriter, r *http.Request) {
diff --git a/unfurl.go b/unfurl.go
new file mode 100644
index 0000000..71dfe39
--- /dev/null
+++ b/unfurl.go
@@ -0,0 +1,213 @@
+package main
+
+import (
+ "errors"
+ "fmt"
+ "html"
+ "io"
+ "net"
+ "net/http"
+ "net/url"
+ "regexp"
+ "strings"
+ "syscall"
+ "time"
+)
+
+// Link (OpenGraph) previews. The server fetches a posted URL only long enough to
+// extract its title/description/image, then caches the result. This is the one
+// place the server sees plaintext URLs (message text stays encrypted); it can be
+// disabled later if that trade-off stops being acceptable.
+
+const (
+ previewFreshTTL = 7 * 24 * 60 * 60 // serve-stale-after, then refresh in the background
+ previewFailTTL = 60 * 60 // retry failed/blank lookups sooner
+ previewMaxBytes = 512 * 1024 // only read the start of the page
+ previewTimeout = 6 * time.Second
+)
+
+// LinkPreview is the cached unfurl result (also the JSON shape sent to clients).
+type LinkPreview struct {
+ URL string `json:"url"`
+ Title string `json:"title"`
+ Description string `json:"description"`
+ Image string `json:"image"`
+ OK bool `json:"ok"`
+ FetchedAt int64 `json:"-"`
+}
+
+// allowPrivatePreviewHosts is flipped on only in tests (so an httptest server on
+// loopback can be fetched); in production the SSRF guard always applies.
+var allowPrivatePreviewHosts = false
+
+// isPublicIP rejects loopback/private/link-local/CGNAT/unspecified addresses so a
+// posted URL can't make the server probe its own network (SSRF).
+func isPublicIP(ip net.IP) bool {
+ if ip == nil {
+ return false
+ }
+ if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() ||
+ ip.IsLinkLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified() {
+ return false
+ }
+ // 100.64.0.0/10 (CGNAT) — not covered by IsPrivate.
+ if v4 := ip.To4(); v4 != nil && v4[0] == 100 && v4[1] >= 64 && v4[1] <= 127 {
+ return false
+ }
+ return true
+}
+
+// ogClient fetches preview pages. Its dialer checks the *resolved* IP of every
+// connection, which also defeats DNS-rebinding (each hop is re-validated).
+var ogClient = &http.Client{
+ Timeout: previewTimeout,
+ Transport: &http.Transport{
+ DialContext: (&net.Dialer{
+ Timeout: previewTimeout,
+ Control: func(network, address string, _ syscall.RawConn) error {
+ if allowPrivatePreviewHosts {
+ return nil
+ }
+ host, _, err := net.SplitHostPort(address)
+ if err != nil {
+ return err
+ }
+ if !isPublicIP(net.ParseIP(host)) {
+ return fmt.Errorf("blocked non-public address %s", address)
+ }
+ return nil
+ },
+ }).DialContext,
+ TLSHandshakeTimeout: previewTimeout,
+ ResponseHeaderTimeout: previewTimeout,
+ DisableKeepAlives: true,
+ },
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ if len(via) >= 5 {
+ return errors.New("too many redirects")
+ }
+ if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
+ return errors.New("non-http redirect")
+ }
+ return nil
+ },
+}
+
+// fetchOG downloads a URL and parses its OpenGraph/HTML metadata.
+func fetchOG(rawURL string) (LinkPreview, error) {
+ pv := LinkPreview{URL: rawURL}
+ req, err := http.NewRequest("GET", rawURL, nil)
+ if err != nil {
+ return pv, err
+ }
+ req.Header.Set("User-Agent", "lchat-linkpreview/1.0 (+https://luan.chat)")
+ req.Header.Set("Accept", "text/html,application/xhtml+xml")
+
+ resp, err := ogClient.Do(req)
+ if err != nil {
+ return pv, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return pv, fmt.Errorf("status %d", resp.StatusCode)
+ }
+ ct := strings.ToLower(resp.Header.Get("Content-Type"))
+ if !strings.Contains(ct, "text/html") && !strings.Contains(ct, "xhtml") {
+ return pv, errors.New("not html")
+ }
+ body, err := io.ReadAll(io.LimitReader(resp.Body, previewMaxBytes))
+ if err != nil {
+ return pv, err
+ }
+ return parseOG(string(body), rawURL), nil
+}
+
+var (
+ metaTagRe = regexp.MustCompile(`(?is)<meta\b[^>]*>`)
+ titleRe = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
+ attrRe = regexp.MustCompile(`(?is)([a-z][a-z0-9:_-]*)\s*=\s*("([^"]*)"|'([^']*)'|([^\s"'>]+))`)
+ tagRe = regexp.MustCompile(`(?is)<[^>]+>`)
+)
+
+func tagAttrs(tag string) map[string]string {
+ out := map[string]string{}
+ for _, m := range attrRe.FindAllStringSubmatch(tag, -1) {
+ val := m[3]
+ if val == "" && m[4] != "" {
+ val = m[4]
+ }
+ if val == "" && m[5] != "" {
+ val = m[5]
+ }
+ out[strings.ToLower(m[1])] = html.UnescapeString(val)
+ }
+ return out
+}
+
+// parseOG extracts a preview from page HTML. og:* wins, then twitter:*, then the
+// <title>/<meta description> fallbacks. A relative og:image is resolved absolute.
+func parseOG(body, pageURL string) LinkPreview {
+ pv := LinkPreview{URL: pageURL}
+ for _, tag := range metaTagRe.FindAllString(body, -1) {
+ a := tagAttrs(tag)
+ key := a["property"]
+ if key == "" {
+ key = a["name"]
+ }
+ content := a["content"]
+ if content == "" {
+ continue
+ }
+ switch strings.ToLower(key) {
+ case "og:title", "twitter:title":
+ if pv.Title == "" {
+ pv.Title = content
+ }
+ case "og:description", "twitter:description", "description":
+ if pv.Description == "" {
+ pv.Description = content
+ }
+ case "og:image", "og:image:url", "og:image:secure_url", "twitter:image", "twitter:image:src":
+ if pv.Image == "" {
+ pv.Image = content
+ }
+ }
+ }
+ if pv.Title == "" {
+ if m := titleRe.FindStringSubmatch(body); m != nil {
+ pv.Title = strings.TrimSpace(html.UnescapeString(tagRe.ReplaceAllString(m[1], "")))
+ }
+ }
+ if pv.Image != "" {
+ pv.Image = resolveImageURL(pageURL, pv.Image)
+ }
+ pv.Title = truncate(strings.TrimSpace(pv.Title), 300)
+ pv.Description = truncate(strings.TrimSpace(pv.Description), 500)
+ pv.OK = pv.Title != "" || pv.Image != ""
+ return pv
+}
+
+// resolveImageURL makes an og:image absolute and keeps only http(s) results.
+func resolveImageURL(pageURL, img string) string {
+ base, err := url.Parse(pageURL)
+ if err != nil {
+ return ""
+ }
+ ref, err := url.Parse(strings.TrimSpace(img))
+ if err != nil {
+ return ""
+ }
+ abs := base.ResolveReference(ref)
+ if abs.Scheme != "http" && abs.Scheme != "https" {
+ return ""
+ }
+ return abs.String()
+}
+
+func truncate(s string, n int) string {
+ r := []rune(s)
+ if len(r) <= n {
+ return s
+ }
+ return strings.TrimSpace(string(r[:n])) + "…"
+}
diff --git a/unfurl_test.go b/unfurl_test.go
new file mode 100644
index 0000000..432c58d
--- /dev/null
+++ b/unfurl_test.go
@@ -0,0 +1,101 @@
+package main
+
+import (
+ "encoding/json"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func TestIsPublicIP(t *testing.T) {
+ blocked := []string{"127.0.0.1", "10.0.0.5", "192.168.1.1", "172.16.0.1", "169.254.1.1", "::1", "0.0.0.0", "100.64.0.1"}
+ for _, s := range blocked {
+ if isPublicIP(net.ParseIP(s)) {
+ t.Errorf("%s should be blocked", s)
+ }
+ }
+ public := []string{"8.8.8.8", "1.1.1.1", "93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"}
+ for _, s := range public {
+ if !isPublicIP(net.ParseIP(s)) {
+ t.Errorf("%s should be allowed", s)
+ }
+ }
+}
+
+func TestParseOG(t *testing.T) {
+ page := `<html><head>
+ <title>Fallback Title</title>
+ <meta property="og:title" content="Real &amp; Title" />
+ <meta name="description" content="A nice description">
+ <meta property="og:image" content="/img/cover.png">
+ </head><body>hi</body></html>`
+ pv := parseOG(page, "https://example.com/post")
+ if pv.Title != "Real & Title" {
+ t.Fatalf("title = %q", pv.Title)
+ }
+ if pv.Description != "A nice description" {
+ t.Fatalf("description = %q", pv.Description)
+ }
+ if pv.Image != "https://example.com/img/cover.png" {
+ t.Fatalf("image = %q (relative not resolved)", pv.Image)
+ }
+ if !pv.OK {
+ t.Fatal("should be OK")
+ }
+}
+
+func TestParseOGTitleFallback(t *testing.T) {
+ pv := parseOG(`<head><title>Just A Title</title></head>`, "https://x.com")
+ if pv.Title != "Just A Title" || !pv.OK {
+ t.Fatalf("fallback title = %q ok=%v", pv.Title, pv.OK)
+ }
+}
+
+func TestUnfurlEndpoint(t *testing.T) {
+ ts, srv := newTestServer(t)
+
+ // The preview target is on loopback (httptest), so allow private hosts here.
+ allowPrivatePreviewHosts = true
+ t.Cleanup(func() { allowPrivatePreviewHosts = false })
+
+ page := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/html")
+ w.Write([]byte(`<head><meta property="og:title" content="Hello OG"><meta property="og:description" content="desc"></head>`))
+ }))
+ t.Cleanup(page.Close)
+
+ resp := adminPost(t, ts.URL+"/api/admin/invites", `{"label":"x"}`)
+ var inv struct {
+ Token string `json:"token"`
+ }
+ json.NewDecoder(resp.Body).Decode(&inv)
+
+ // No token → 403.
+ r, _ := http.Get(ts.URL + "/api/unfurl?url=" + page.URL)
+ if r.StatusCode != http.StatusForbidden {
+ t.Fatalf("unfurl no token: want 403, got %d", r.StatusCode)
+ }
+
+ // Bad URL → 400.
+ r, _ = http.Get(ts.URL + "/api/unfurl?token=" + inv.Token + "&url=notaurl")
+ if r.StatusCode != http.StatusBadRequest {
+ t.Fatalf("unfurl bad url: want 400, got %d", r.StatusCode)
+ }
+
+ // Good fetch → preview returned and cached.
+ r, _ = http.Get(ts.URL + "/api/unfurl?token=" + inv.Token + "&url=" + page.URL)
+ if r.StatusCode != http.StatusOK {
+ t.Fatalf("unfurl: %d", r.StatusCode)
+ }
+ var pv LinkPreview
+ json.NewDecoder(r.Body).Decode(&pv)
+ if !pv.OK || pv.Title != "Hello OG" {
+ t.Fatalf("preview = %+v", pv)
+ }
+
+ // It should now be in the cache.
+ if _, ok, _ := srv.db.getLinkPreview(page.URL); !ok {
+ t.Fatal("preview was not cached")
+ }
+}