eleven / ocho Public

Download this file

1	package bot
2
3	// The fetch capability: the one place a bot touches the public web, used by
4	// task runs (task.go). Deliberately narrow — HTTPS GET only, against a
5	// per-rule allowlist declared in the recipe (a task with no allowlist has no
6	// network), with the same private-address blocking as the server's unfurler.
7	// The sites fetched see the sidecar box's address; that widening is disclosed
8	// in the design doc and in the consent card a builder shows before creating a
9	// bot that uses this.
10
11	import (
12	"errors"
13	"fmt"
14	"io"
15	"net"
16	"net/http"
17	"net/url"
18	"os"
19	"strings"
20	"syscall"
21	"time"
22	)
23
24	// maxFetchBytes caps one fetched body. Feeds and article pages fit far below
25	// this; anything bigger gets truncated, not rejected — a partial feed still
26	// summarises.
27	const maxFetchBytes = 512 * 1024
28
29	const fetchTimeout = 20 * time.Second
30
31	// allowPrivateFetch lets tests fetch from loopback httptest servers. Never set
32	// in production units.
33	func allowPrivateFetch() bool { return os.Getenv("LCHAT_BOT_ALLOW_PRIVATE_FETCH") == "1" }
34
35	// checkFetchURL enforces scheme + allowlist. Allow entries are URL prefixes
36	// ("https://www.rte.ie/feeds/"); a redirect target is re-checked with the same
37	// rule, so a fetch can never wander off the list. The apex↔www dance counts
38	// as the same site (feed discovery already treats it so): a homepage-fallback
39	// entry like "https://irishtimes.com" must survive the site's own 301 to
40	// www.irishtimes.com, or every fetch dies on its first redirect.
41	func checkFetchURL(raw string, allow []string) error {
42	u, err := url.Parse(raw)
43	if err != nil {
44	return fmt.Errorf("bad url: %w", err)
45	}
46	scheme := "https"
47	if allowPrivateFetch() {
48	scheme = "http" // loopback test servers are plain http
49	}
50	if u.Scheme != "https" && u.Scheme != scheme {
51	return fmt.Errorf("only https urls may be fetched")
52	}
53	for _, candidate := range []string{raw, toggleWWW(u)} {
54	if candidate == "" {
55	continue
56	}
57	for _, a := range allow {
58	if allowPrefixMatch(candidate, a) {
59	return nil
60	}
61	}
62	}
63	return fmt.Errorf("url not in this bot's allowlist")
64	}
65
66	// allowPrefixMatch reports whether a URL falls under one allow entry. Not a
67	// bare HasPrefix: a bare-host entry ("https://irishtimes.com") must not match
68	// a LONGER host ("https://irishtimes.com.evil.com" — and the MODEL picks
69	// fetch URLs, so the boundary is load-bearing). Unless the entry already ends
70	// inside its path or query, the match must stop at a URL boundary character.
71	func allowPrefixMatch(raw, entry string) bool {
72	if !strings.HasPrefix(raw, entry) {
73	return false
74	}
75	if len(raw) == len(entry) || strings.HasSuffix(entry, "/") || strings.Contains(entry, "?") {
76	return true
77	}
78	switch raw[len(entry)] {
79	case '/', '?', '#':
80	return true
81	}
82	return false
83	}
84
85	// toggleWWW is u with its host's leading "www." flipped (added or removed) —
86	// the same-site variant the allowlist tolerates. "" when there's no host.
87	func toggleWWW(u *url.URL) string {
88	if u.Host == "" {
89	return ""
90	}
91	v := *u
92	if strings.HasPrefix(v.Host, "www.") {
93	v.Host = strings.TrimPrefix(v.Host, "www.")
94	} else {
95	v.Host = "www." + v.Host
96	}
97	return v.String()
98	}
99
100	// privateIP reports whether an address must not be dialled: loopback,
101	// link-local, RFC1918/ULA — the standard SSRF surface. Checked in the dialer's
102	// Control hook, i.e. against the address actually being connected to (after
103	// DNS), so a hostname that resolves privately is caught too.
104	func privateIP(ip net.IP) bool {
105	return ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
106	ip.IsPrivate() || ip.IsUnspecified()
107	}
108
109	func fetchClient(allow []string) *http.Client {
110	dialer := &net.Dialer{
111	Timeout: 10 * time.Second,
112	Control: func(network, address string, _ syscall.RawConn) error {
113	if allowPrivateFetch() {
114	return nil
115	}
116	host, _, err := net.SplitHostPort(address)
117	if err != nil {
118	return err
119	}
120	if ip := net.ParseIP(host); ip == nil || privateIP(ip) {
121	return errors.New("private address blocked")
122	}
123	return nil
124	},
125	}
126	return &http.Client{
127	Timeout: fetchTimeout,
128	Transport: &http.Transport{DialContext: dialer.DialContext},
129	CheckRedirect: func(req *http.Request, via []*http.Request) error {
130	if len(via) >= 3 {
131	return errors.New("too many redirects")
132	}
133	// A redirect is a new URL: hold it to the same allowlist + scheme
134	// rules as the original.
135	return checkFetchURL(req.URL.String(), allow)
136	},
137	}
138	}
139
140	// fetchURL GETs one allowlisted URL and returns its body as text (truncated at
141	// maxFetchBytes). Errors are returned as values — the task loop feeds them
142	// back to the model as tool_result content, so a dead feed degrades to an
143	// honest "couldn't fetch" instead of a silent gap.
144	func fetchURL(raw string, allow []string) (string, error) {
145	if err := checkFetchURL(raw, allow); err != nil {
146	return "", err
147	}
148	req, err := http.NewRequest(http.MethodGet, raw, nil)
149	if err != nil {
150	return "", err
151	}
152	req.Header.Set("User-Agent", "lchat-bot/1.0 (+https://elevenmessenger.com)")
153	req.Header.Set("Accept", "application/rss+xml, application/atom+xml, text/html, text/plain, */*")
154	resp, err := fetchClient(allow).Do(req)
155	if err != nil {
156	return "", err
157	}
158	defer resp.Body.Close()
159	if resp.StatusCode != http.StatusOK {
160	return "", fmt.Errorf("fetch %s: status %d", raw, resp.StatusCode)
161	}
162	body, err := io.ReadAll(io.LimitReader(resp.Body, maxFetchBytes))
163	if err != nil {
164	return "", err
165	}
166	return string(body), nil
167	}
168