| 1 | // Package notion speaks the Notion API and nothing else. |
| 2 | // |
| 3 | // It holds no database handle, no sealer and no app type. That boundary |
| 4 | // is the testable one: everything here is exercised against a fake HTTP |
| 5 | // server, and the app layer above never parses JSON. |
| 6 | // |
| 7 | // This is the ONLY package in memoria permitted to reach the network. |
| 8 | // TestNoOutboundNetworkInAppCode forbids it everywhere else and exempts |
| 9 | // exactly this directory; the exemption is paid for by the host pin |
| 10 | // below, which is a runtime guard rather than a grep over source. |
| 11 | package notion |
| 12 | |
| 13 | import ( |
| 14 | "bytes" |
| 15 | "context" |
| 16 | "encoding/json" |
| 17 | "errors" |
| 18 | "fmt" |
| 19 | "io" |
| 20 | "net" |
| 21 | "net/http" |
| 22 | "strconv" |
| 23 | "time" |
| 24 | ) |
| 25 | |
| 26 | // BaseURL is where the API lives, and it is a CONSTANT. |
| 27 | // |
| 28 | // Nothing configurable feeds it. The token travels in an Authorization |
| 29 | // header, so a base URL built from stored or submitted input would make |
| 30 | // memoria a token courier for whoever could write that input. |
| 31 | const BaseURL = "https://api.notion.com" |
| 32 | |
| 33 | // APIVersion is pinned, and the pin is load-bearing rather than tidy. |
| 34 | // |
| 35 | // Version 2025-09-03 changed the object model incompatibly: a database |
| 36 | // now contains data sources, a row's parent is a data_source_id, and |
| 37 | // /v1/databases/{id}/query was replaced. 2026-03-11 further renamed |
| 38 | // `archived` to `in_trash` and the `transcription` block type to |
| 39 | // `meeting_notes`. An unpinned client is one Notion release away from |
| 40 | // extracting nothing and reporting success. |
| 41 | const APIVersion = "2026-03-11" |
| 42 | |
| 43 | // Rate limits, from Notion's published request limits: an average of |
| 44 | // three requests per second per connection, bursts allowed. We pace |
| 45 | // below the average rather than at it, because the second limit is |
| 46 | // per-workspace and shared with every other connection the team runs — |
| 47 | // so the budget we are actually spending is not ours alone. |
| 48 | const defaultInterval = 350 * time.Millisecond |
| 49 | |
| 50 | // retryBudget bounds retries. On exhaustion the client returns an error; |
| 51 | // it NEVER returns what it managed to collect. A partial result that |
| 52 | // looks like a whole one is how an import reports success while missing |
| 53 | // pages. |
| 54 | const retryBudget = 5 |
| 55 | |
| 56 | // maxRetryAfter caps how long we will honour a Retry-After. |
| 57 | // |
| 58 | // Notion sends it in seconds and nothing bounds what it may ask for. An |
| 59 | // uncapped honour of "Retry-After: 86400" parks the run for a day, and |
| 60 | // because the goroutine is also what proves the run is alive, a wait |
| 61 | // past the staleness window gets the run reclaimed and a SECOND one |
| 62 | // started on the same ledger. Past this we give up and say so, which is |
| 63 | // recoverable; sleeping for a day is not. |
| 64 | const maxRetryAfter = 2 * time.Minute |
| 65 | |
| 66 | // Code is a closed set of error codes. It is a code and never a message |
| 67 | // because Notion's own error bodies carry content: a 404 reads "Could |
| 68 | // not find database with ID: be907abe-… shared with your connection "My |
| 69 | // Connection"", which is a page id and a workspace name one %w away from |
| 70 | // a log line AGENTS.md forbids. |
| 71 | type Code string |
| 72 | |
| 73 | const ( |
| 74 | CodeUnauthorized Code = "unauthorized" |
| 75 | CodeRateLimited Code = "rate_limited" |
| 76 | CodeTraversalFailed Code = "traversal_failed" |
| 77 | CodeExtractFailed Code = "extract_failed" |
| 78 | CodeNetwork Code = "network" |
| 79 | CodeForbiddenHost Code = "forbidden_host" |
| 80 | ) |
| 81 | |
| 82 | // Error carries a code and an HTTP status. Deliberately no message and |
| 83 | // no body: a status is a fact about the transport, a body is content. |
| 84 | type Error struct { |
| 85 | Code Code |
| 86 | Status int |
| 87 | } |
| 88 | |
| 89 | func (e *Error) Error() string { |
| 90 | if e.Status != 0 { |
| 91 | return fmt.Sprintf("notion: %s (http %d)", e.Code, e.Status) |
| 92 | } |
| 93 | return "notion: " + string(e.Code) |
| 94 | } |
| 95 | |
| 96 | // CodeOf returns the code of an error, or CodeNetwork. |
| 97 | func CodeOf(err error) Code { |
| 98 | var e *Error |
| 99 | if errors.As(err, &e) { |
| 100 | return e.Code |
| 101 | } |
| 102 | return CodeNetwork |
| 103 | } |
| 104 | |
| 105 | // pinnedTransport refuses every host but the one it was built for. |
| 106 | // |
| 107 | // This is the runtime half of the network gate. Go already drops the |
| 108 | // Authorization header across a cross-host redirect, which is not |
| 109 | // enough: a SAME-host redirect keeps it, and a base URL that had been |
| 110 | // tampered with would never be cross-host in the first place. So the |
| 111 | // check is on every request, not only on redirects — and redirects are |
| 112 | // refused outright below. |
| 113 | // |
| 114 | // The scheme is pinned alongside the host rather than special-cased to |
| 115 | // "https unless loopback". The special case looked harmless and was not: |
| 116 | // it rejected a wrong HOST that also happened to be non-loopback, which |
| 117 | // meant the gate for the host pin passed with the host pin deleted — the |
| 118 | // neighbouring-error-path failure AGENTS.md opens with. Two fields, two |
| 119 | // checks, one mutation each. |
| 120 | type pinnedTransport struct { |
| 121 | host string |
| 122 | scheme string |
| 123 | next http.RoundTripper |
| 124 | } |
| 125 | |
| 126 | func (p *pinnedTransport) RoundTrip(r *http.Request) (*http.Response, error) { |
| 127 | if r.URL.Hostname() != p.host { |
| 128 | return nil, &Error{Code: CodeForbiddenHost} |
| 129 | } |
| 130 | if r.URL.Scheme != p.scheme { |
| 131 | return nil, &Error{Code: CodeForbiddenHost} |
| 132 | } |
| 133 | return p.next.RoundTrip(r) |
| 134 | } |
| 135 | |
| 136 | // Client is a Notion API client. |
| 137 | type Client struct { |
| 138 | token string |
| 139 | base string |
| 140 | http *http.Client |
| 141 | interval time.Duration |
| 142 | last time.Time |
| 143 | now func() time.Time |
| 144 | sleep func(time.Duration) |
| 145 | } |
| 146 | |
| 147 | // New builds a client against the real API. |
| 148 | func New(token string) (*Client, error) { |
| 149 | return newClient(token, BaseURL, defaultInterval) |
| 150 | } |
| 151 | |
| 152 | // NewForTest builds a client against an arbitrary base URL, so the |
| 153 | // package can be driven by an httptest server. |
| 154 | // |
| 155 | // Named so it cannot be reached from production by accident, exactly as |
| 156 | // memoria.AppForTest is, and held to the same standard: the host pin is |
| 157 | // still applied — to the test server's host — so the transport's refusal |
| 158 | // path is the one under test rather than a bypassed one. |
| 159 | func NewForTest(token, base string) (*Client, error) { |
| 160 | return newClient(token, base, 0) |
| 161 | } |
| 162 | |
| 163 | func newClient(token, base string, interval time.Duration) (*Client, error) { |
| 164 | if token == "" { |
| 165 | return nil, &Error{Code: CodeUnauthorized} |
| 166 | } |
| 167 | host, scheme, err := hostOf(base) |
| 168 | if err != nil { |
| 169 | return nil, err |
| 170 | } |
| 171 | return &Client{ |
| 172 | token: token, |
| 173 | base: base, |
| 174 | interval: interval, |
| 175 | now: time.Now, |
| 176 | sleep: time.Sleep, |
| 177 | http: &http.Client{ |
| 178 | Timeout: 60 * time.Second, |
| 179 | Transport: &pinnedTransport{host: host, scheme: scheme, next: http.DefaultTransport}, |
| 180 | CheckRedirect: func(*http.Request, []*http.Request) error { |
| 181 | // Refuse every redirect. There is no legitimate one on |
| 182 | // this API, and following one is the cheapest way to |
| 183 | // hand a bearer token to somewhere it was not meant for. |
| 184 | return &Error{Code: CodeForbiddenHost} |
| 185 | }, |
| 186 | }, |
| 187 | }, nil |
| 188 | } |
| 189 | |
| 190 | func hostOf(base string) (host, scheme string, err error) { |
| 191 | req, err := http.NewRequest(http.MethodGet, base, nil) |
| 192 | if err != nil { |
| 193 | return "", "", &Error{Code: CodeForbiddenHost} |
| 194 | } |
| 195 | h, sc := req.URL.Hostname(), req.URL.Scheme |
| 196 | if h == "" || sc == "" { |
| 197 | return "", "", &Error{Code: CodeForbiddenHost} |
| 198 | } |
| 199 | return h, sc, nil |
| 200 | } |
| 201 | |
| 202 | // pace holds the client below the published rate limit. |
| 203 | func (c *Client) pace() { |
| 204 | if c.interval <= 0 { |
| 205 | return |
| 206 | } |
| 207 | if wait := c.interval - c.now().Sub(c.last); wait > 0 { |
| 208 | c.sleep(wait) |
| 209 | } |
| 210 | c.last = c.now() |
| 211 | } |
| 212 | |
| 213 | // post issues one API call and decodes into out, retrying only what |
| 214 | // Notion says is retryable. |
| 215 | func (c *Client) post(ctx context.Context, path string, body any, out any) error { |
| 216 | return c.call(ctx, http.MethodPost, path, body, out) |
| 217 | } |
| 218 | |
| 219 | func (c *Client) get(ctx context.Context, path string, out any) error { |
| 220 | return c.call(ctx, http.MethodGet, path, nil, out) |
| 221 | } |
| 222 | |
| 223 | func (c *Client) call(ctx context.Context, method, path string, body, out any) error { |
| 224 | var payload []byte |
| 225 | if body != nil { |
| 226 | var err error |
| 227 | if payload, err = json.Marshal(body); err != nil { |
| 228 | return &Error{Code: CodeExtractFailed} |
| 229 | } |
| 230 | } |
| 231 | for attempt := 0; ; attempt++ { |
| 232 | c.pace() |
| 233 | |
| 234 | var rdr io.Reader |
| 235 | if payload != nil { |
| 236 | rdr = bytes.NewReader(payload) |
| 237 | } |
| 238 | req, err := http.NewRequestWithContext(ctx, method, c.base+path, rdr) |
| 239 | if err != nil { |
| 240 | return &Error{Code: CodeNetwork} |
| 241 | } |
| 242 | req.Header.Set("Authorization", "Bearer "+c.token) |
| 243 | req.Header.Set("Notion-Version", APIVersion) |
| 244 | if payload != nil { |
| 245 | req.Header.Set("Content-Type", "application/json") |
| 246 | } |
| 247 | |
| 248 | res, err := c.http.Do(req) |
| 249 | if err != nil { |
| 250 | // A pinned-host or redirect refusal is not retryable: it is |
| 251 | // a statement about where this request was going. |
| 252 | var pe *Error |
| 253 | if errors.As(err, &pe) && pe.Code == CodeForbiddenHost { |
| 254 | return pe |
| 255 | } |
| 256 | if ctx.Err() != nil { |
| 257 | return &Error{Code: CodeNetwork} |
| 258 | } |
| 259 | var ne net.Error |
| 260 | if attempt < retryBudget && errors.As(err, &ne) { |
| 261 | if !c.backoff(ctx, attempt, 0) { |
| 262 | return &Error{Code: CodeNetwork} |
| 263 | } |
| 264 | continue |
| 265 | } |
| 266 | return &Error{Code: CodeNetwork} |
| 267 | } |
| 268 | |
| 269 | retryable, retryAfter := classify(res.StatusCode, res.Header.Get("Retry-After")) |
| 270 | if retryable { |
| 271 | res.Body.Close() |
| 272 | if attempt < retryBudget { |
| 273 | if !c.backoff(ctx, attempt, retryAfter) { |
| 274 | return &Error{Code: CodeNetwork} |
| 275 | } |
| 276 | continue |
| 277 | } |
| 278 | // Budget exhausted. Return the error — never the partial |
| 279 | // work. See retryBudget. |
| 280 | code := CodeTraversalFailed |
| 281 | if res.StatusCode == http.StatusTooManyRequests || res.StatusCode == 529 { |
| 282 | code = CodeRateLimited |
| 283 | } |
| 284 | return &Error{Code: code, Status: res.StatusCode} |
| 285 | } |
| 286 | |
| 287 | if res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden { |
| 288 | res.Body.Close() |
| 289 | return &Error{Code: CodeUnauthorized, Status: res.StatusCode} |
| 290 | } |
| 291 | if res.StatusCode < 200 || res.StatusCode > 299 { |
| 292 | res.Body.Close() |
| 293 | return &Error{Code: CodeTraversalFailed, Status: res.StatusCode} |
| 294 | } |
| 295 | |
| 296 | // Cap the body. An unbounded ReadAll on a response we do not |
| 297 | // control is a memory bug waiting for a bad day. |
| 298 | raw, err := io.ReadAll(io.LimitReader(res.Body, 32<<20)) |
| 299 | res.Body.Close() |
| 300 | if err != nil { |
| 301 | return &Error{Code: CodeNetwork} |
| 302 | } |
| 303 | if out == nil { |
| 304 | return nil |
| 305 | } |
| 306 | if err := json.Unmarshal(raw, out); err != nil { |
| 307 | // A body we cannot parse is NOT an empty result. Returning |
| 308 | // nil here would turn a malformed page into "no children", |
| 309 | // which is the plausible-empty-list failure this codebase |
| 310 | // ranks worst. |
| 311 | return &Error{Code: CodeTraversalFailed, Status: res.StatusCode} |
| 312 | } |
| 313 | return nil |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | // classify says whether a status is worth retrying, and for how long. |
| 318 | // |
| 319 | // 429 and 529 both carry Retry-After in integer seconds — 529 is |
| 320 | // Notion's "service_overload" and is documented to be retried the same |
| 321 | // way as a 429, which is easy to miss because it is not a standard code. |
| 322 | // 5xx are retried because every request this package makes is a read. |
| 323 | func classify(status int, retryAfter string) (bool, time.Duration) { |
| 324 | switch status { |
| 325 | case http.StatusTooManyRequests, 529: |
| 326 | if s, err := strconv.Atoi(retryAfter); err == nil && s >= 0 { |
| 327 | d := time.Duration(s) * time.Second |
| 328 | if d > maxRetryAfter { |
| 329 | d = maxRetryAfter |
| 330 | } |
| 331 | return true, d |
| 332 | } |
| 333 | return true, 0 |
| 334 | case http.StatusInternalServerError, http.StatusBadGateway, |
| 335 | http.StatusServiceUnavailable, http.StatusGatewayTimeout: |
| 336 | return true, 0 |
| 337 | } |
| 338 | return false, 0 |
| 339 | } |
| 340 | |
| 341 | // backoff waits, and stops waiting if the context is done. |
| 342 | // |
| 343 | // c.sleep used to be a bare time.Sleep, so a cancelled run kept sleeping |
| 344 | // — for up to maxRetryAfter, per attempt, per page. |
| 345 | func (c *Client) backoff(ctx context.Context, attempt int, after time.Duration) bool { |
| 346 | if c.interval <= 0 && after == 0 { |
| 347 | return ctx.Err() == nil // tests: no real waiting |
| 348 | } |
| 349 | d := after |
| 350 | if d <= 0 { |
| 351 | d = time.Duration(1<<attempt) * 200 * time.Millisecond |
| 352 | } |
| 353 | if d > maxRetryAfter { |
| 354 | d = maxRetryAfter |
| 355 | } |
| 356 | t := time.NewTimer(d) |
| 357 | defer t.Stop() |
| 358 | select { |
| 359 | case <-ctx.Done(): |
| 360 | return false |
| 361 | case <-t.C: |
| 362 | return true |
| 363 | } |
| 364 | } |
| 365 | |
amadan