| 1 | package notion |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "net/url" |
| 7 | "strings" |
| 8 | ) |
| 9 | |
| 10 | // Ref is one page memoria may hold: a coordinate plus the ancestry it |
| 11 | // was reached through. |
| 12 | type Ref struct { |
| 13 | // ID is the Notion page id. A coordinate, and the caller seals it. |
| 14 | ID string |
| 15 | // Title and Path are CONTENT. Never log them. |
| 16 | Title string |
| 17 | Path []string |
| 18 | // EditedTime is Notion's last_edited_time, the change detector. |
| 19 | EditedTime string |
| 20 | // Kind distinguishes an ordinary page from a data-source row, which |
| 21 | // the extractor renders differently. |
| 22 | Kind Kind |
| 23 | // DataSourceID is set when Kind is KindDataSource: the thing to |
| 24 | // query for rows. |
| 25 | DataSourceID string |
| 26 | // Properties is a database row's raw property map. CONTENT — never |
| 27 | // logged — and carried here because a row's body is usually empty: |
| 28 | // the information is in the columns. Without it a forty-row database |
| 29 | // becomes forty blank documents, which the design warned about and |
| 30 | // the first implementation shipped anyway, with the renderer written |
| 31 | // and never called. |
| 32 | Properties json.RawMessage |
| 33 | } |
| 34 | |
| 35 | type Kind string |
| 36 | |
| 37 | const ( |
| 38 | KindPage Kind = "page" |
| 39 | KindDataSource Kind = "data_source" |
| 40 | ) |
| 41 | |
| 42 | // Content is one page's extracted body plus what lies beneath it. |
| 43 | type Content struct { |
| 44 | Markdown string |
| 45 | // Children are the pages and data sources nested directly under this |
| 46 | // one, with their Path already resolved — traversal knows the |
| 47 | // ancestry because it arrived from the parent. |
| 48 | Children []Ref |
| 49 | // UnreadBlocks counts blocks of a type this package does not render. |
| 50 | // Reported rather than swallowed: a page that came in thin then has |
| 51 | // a reason attached instead of looking like a thin page. |
| 52 | UnreadBlocks int |
| 53 | } |
| 54 | |
| 55 | // list is one page of a paginated Notion response. |
| 56 | type list struct { |
| 57 | Results []json.RawMessage `json:"results"` |
| 58 | NextCursor *string `json:"next_cursor"` |
| 59 | HasMore bool `json:"has_more"` |
| 60 | } |
| 61 | |
| 62 | // object is the shared head of every Notion object we read. |
| 63 | type object struct { |
| 64 | Object string `json:"object"` |
| 65 | ID string `json:"id"` |
| 66 | InTrash bool `json:"in_trash"` |
| 67 | Parent parent `json:"parent"` |
| 68 | Properties json.RawMessage `json:"properties"` |
| 69 | Title []richText `json:"title"` |
| 70 | Name string `json:"name"` |
| 71 | LastEdited string `json:"last_edited_time"` |
| 72 | } |
| 73 | |
| 74 | type parent struct { |
| 75 | Type string `json:"type"` |
| 76 | PageID string `json:"page_id"` |
| 77 | DatabaseID string `json:"database_id"` |
| 78 | DataSourceID string `json:"data_source_id"` |
| 79 | BlockID string `json:"block_id"` |
| 80 | Workspace bool `json:"workspace"` |
| 81 | } |
| 82 | |
| 83 | // id returns the parent's id whatever kind it is, so "is my parent in |
| 84 | // this set?" is one lookup rather than four. |
| 85 | func (p parent) id() string { |
| 86 | switch { |
| 87 | case p.PageID != "": |
| 88 | return p.PageID |
| 89 | case p.DataSourceID != "": |
| 90 | return p.DataSourceID |
| 91 | case p.DatabaseID != "": |
| 92 | return p.DatabaseID |
| 93 | case p.BlockID != "": |
| 94 | return p.BlockID |
| 95 | } |
| 96 | return "" |
| 97 | } |
| 98 | |
| 99 | type richText struct { |
| 100 | PlainText string `json:"plain_text"` |
| 101 | } |
| 102 | |
| 103 | func plain(rt []richText) string { |
| 104 | var b strings.Builder |
| 105 | for _, r := range rt { |
| 106 | b.WriteString(r.PlainText) |
| 107 | } |
| 108 | return b.String() |
| 109 | } |
| 110 | |
| 111 | // paginate walks every page of a cursor-paginated endpoint and applies |
| 112 | // the total-decoder rules. It returns everything or it returns an error; |
| 113 | // there is no partial result, because a partial result that looks whole |
| 114 | // is how an import reports success while missing pages. |
| 115 | func (c *Client) paginate(ctx context.Context, method, path string, body map[string]any, fn func(json.RawMessage) error) error { |
| 116 | seen := map[string]bool{} |
| 117 | cursor := "" |
| 118 | for { |
| 119 | var page list |
| 120 | var err error |
| 121 | if method == "POST" { |
| 122 | b := map[string]any{"page_size": 100} |
| 123 | for k, v := range body { |
| 124 | b[k] = v |
| 125 | } |
| 126 | if cursor != "" { |
| 127 | b["start_cursor"] = cursor |
| 128 | } |
| 129 | err = c.post(ctx, path, b, &page) |
| 130 | } else { |
| 131 | // Escaped: the cursor is an opaque string from a response, |
| 132 | // and splicing it raw into a query string is how a stray & or |
| 133 | // # silently truncates the request. |
| 134 | p := path + "?page_size=100" |
| 135 | if cursor != "" { |
| 136 | p += "&start_cursor=" + url.QueryEscape(cursor) |
| 137 | } |
| 138 | err = c.get(ctx, p, &page) |
| 139 | } |
| 140 | if err != nil { |
| 141 | return err |
| 142 | } |
| 143 | for _, r := range page.Results { |
| 144 | if err := fn(r); err != nil { |
| 145 | return err |
| 146 | } |
| 147 | } |
| 148 | if !page.HasMore { |
| 149 | return nil |
| 150 | } |
| 151 | // has_more with no cursor is unfinishable. Treating it as the |
| 152 | // end would silently truncate the enumeration. |
| 153 | if page.NextCursor == nil || *page.NextCursor == "" { |
| 154 | return &Error{Code: CodeTraversalFailed} |
| 155 | } |
| 156 | // A repeating cursor is a loop. Without this the run never |
| 157 | // finishes and never says why. |
| 158 | if seen[*page.NextCursor] { |
| 159 | return &Error{Code: CodeTraversalFailed} |
| 160 | } |
| 161 | seen[*page.NextCursor] = true |
| 162 | cursor = *page.NextCursor |
| 163 | } |
| 164 | } |
| 165 | |
| 166 | // Roots returns the pages and data sources to traverse from. |
| 167 | // |
| 168 | // Search is used HERE AND ONLY HERE, and only for what Notion guarantees |
| 169 | // it does: "any pages or databases that are directly shared with a |
| 170 | // connection are guaranteed to be returned". Notion documents that |
| 171 | // search is NOT an exhaustive enumerator — "search is not guaranteed to |
| 172 | // return everything" — and returns no field saying a result was partial, |
| 173 | // so a run that paged cleanly to the end could be missing pages and |
| 174 | // would report a complete import. Everything below a root is found by |
| 175 | // traversal instead, which enumerates a parent's actual children. |
| 176 | // |
| 177 | // A result whose parent is also in the result set is not a root: it will |
| 178 | // be reached by traversal, with its ancestry known. A result whose |
| 179 | // parent is absent IS a root, even if it is nested in Notion — we cannot |
| 180 | // see above it, so its ancestry starts here and the status page says how |
| 181 | // many roots were found. |
| 182 | func (c *Client) Roots(ctx context.Context) (roots, orphans []Ref, err error) { |
| 183 | var objs []object |
| 184 | err = c.paginate(ctx, "POST", "/v1/search", nil, func(raw json.RawMessage) error { |
| 185 | var o object |
| 186 | if err := json.Unmarshal(raw, &o); err != nil { |
| 187 | return &Error{Code: CodeTraversalFailed} |
| 188 | } |
| 189 | objs = append(objs, o) |
| 190 | return nil |
| 191 | }) |
| 192 | if err != nil { |
| 193 | return nil, nil, err |
| 194 | } |
| 195 | |
| 196 | present := make(map[string]bool, len(objs)) |
| 197 | for _, o := range objs { |
| 198 | present[o.ID] = true |
| 199 | } |
| 200 | |
| 201 | for _, o := range objs { |
| 202 | if o.InTrash { |
| 203 | continue |
| 204 | } |
| 205 | if pid := o.Parent.id(); pid != "" && present[pid] { |
| 206 | continue // reachable by traversal, with its ancestry known |
| 207 | } |
| 208 | r := Ref{ID: o.ID, EditedTime: o.LastEdited, Kind: KindPage} |
| 209 | switch o.Object { |
| 210 | case "data_source": |
| 211 | r.Kind, r.DataSourceID = KindDataSource, o.ID |
| 212 | r.Title = firstNonEmpty(plain(o.Title), o.Name) |
| 213 | case "database": |
| 214 | // A database is a container of data sources, not a document. |
| 215 | // Its data sources are resolved when it is traversed. |
| 216 | r.Kind, r.DataSourceID = KindDataSource, o.ID |
| 217 | r.Title = firstNonEmpty(plain(o.Title), o.Name) |
| 218 | default: |
| 219 | r.Title = titleOf(o) |
| 220 | } |
| 221 | if r.Title == "" { |
| 222 | r.Title = "Untitled" |
| 223 | } |
| 224 | // A result whose parent is the WORKSPACE is a true root: nothing |
| 225 | // sits above it and its ancestry genuinely starts here. |
| 226 | // |
| 227 | // Anything else reaching this point is an ORPHAN — its parent |
| 228 | // exists but search did not return it. Two ways that happens, and |
| 229 | // both are ordinary rather than exotic: a page nested inside a |
| 230 | // toggle or a bulleted item has a `block_id` parent, and search |
| 231 | // never returns blocks, so `present[parent]` is always false for |
| 232 | // it; and search's index is eventually consistent, so a middle |
| 233 | // ancestor can simply be missing from one response. |
| 234 | // |
| 235 | // Treating an orphan as a root is how a nested page gets ingested |
| 236 | // with EMPTY ancestry. Worse, it used to win: roots were queued |
| 237 | // first, so the orphan was visited and sealed with no path before |
| 238 | // the traversal reached it legitimately, and the correct |
| 239 | // `Handbook › Onboarding` Ref was then discarded as already |
| 240 | // visited. On the next run search might include the ancestor, the |
| 241 | // path print would change, and the page would flap between two |
| 242 | // ancestries with a re-ingest each time. |
| 243 | // |
| 244 | // So orphans come back separately and the caller walks them only |
| 245 | // after the real traversal has drained. Found by a code review; |
| 246 | // the fake had a searchLeaks switch built for exactly this case |
| 247 | // and no test using it. |
| 248 | if o.Parent.Type == "workspace" || o.Parent.Workspace { |
| 249 | roots = append(roots, r) |
| 250 | } else { |
| 251 | orphans = append(orphans, r) |
| 252 | } |
| 253 | } |
| 254 | return roots, orphans, nil |
| 255 | } |
| 256 | |
| 257 | // titleOf pulls a page's title out of its properties. The title property |
| 258 | // is identified by its TYPE, not its name: Notion lets it be renamed and |
| 259 | // commonly is ("Name", "Task", "Project"). |
| 260 | func titleOf(o object) string { |
| 261 | if len(o.Title) > 0 { |
| 262 | return plain(o.Title) |
| 263 | } |
| 264 | if len(o.Properties) == 0 { |
| 265 | return "" |
| 266 | } |
| 267 | var props map[string]struct { |
| 268 | Type string `json:"type"` |
| 269 | Title []richText `json:"title"` |
| 270 | } |
| 271 | if err := json.Unmarshal(o.Properties, &props); err != nil { |
| 272 | return "" |
| 273 | } |
| 274 | for _, p := range props { |
| 275 | if p.Type == "title" { |
| 276 | return plain(p.Title) |
| 277 | } |
| 278 | } |
| 279 | return "" |
| 280 | } |
| 281 | |
| 282 | func firstNonEmpty(vals ...string) string { |
| 283 | for _, v := range vals { |
| 284 | if strings.TrimSpace(v) != "" { |
| 285 | return v |
| 286 | } |
| 287 | } |
| 288 | return "" |
| 289 | } |
| 290 | |
| 291 | // Fetch reads one page: its body as Markdown, and the pages nested |
| 292 | // directly under it with their ancestry already resolved. |
| 293 | func (c *Client) Fetch(ctx context.Context, ref Ref) (*Content, error) { |
| 294 | if ref.Kind == KindDataSource { |
| 295 | return c.fetchDataSource(ctx, ref) |
| 296 | } |
| 297 | blocks, err := c.blocks(ctx, ref.ID) |
| 298 | if err != nil { |
| 299 | return nil, err |
| 300 | } |
| 301 | childPath := append(append([]string{}, ref.Path...), ref.Title) |
| 302 | md, kids, unread := render(blocks, childPath) |
| 303 | // A database row's columns lead its body. They are usually the whole |
| 304 | // of what the row says. |
| 305 | if props := Properties(ref.Properties); props != "" { |
| 306 | if md == "" { |
| 307 | md = props |
| 308 | } else { |
| 309 | md = props + "\n\n" + md |
| 310 | } |
| 311 | } |
| 312 | return &Content{Markdown: md, Children: kids, UnreadBlocks: unread}, nil |
| 313 | } |
| 314 | |
| 315 | // firstProperty returns the first non-empty rendered property value, for |
| 316 | // naming a row whose title property is blank. |
| 317 | func firstProperty(raw json.RawMessage) string { |
| 318 | rendered := Properties(raw) |
| 319 | for _, line := range strings.Split(rendered, "\n") { |
| 320 | if _, v, ok := strings.Cut(line, ": "); ok && v != "" { |
| 321 | return v |
| 322 | } |
| 323 | } |
| 324 | return "" |
| 325 | } |
| 326 | |
| 327 | // fetchDataSource turns a database into its rows. |
| 328 | // |
| 329 | // The database itself is not a document — it is a container — so its |
| 330 | // Markdown is empty and its rows are its children. Under API version |
| 331 | // 2025-09-03 and later a database holds one or more DATA SOURCES and |
| 332 | // rows are queried from those, not from the database. |
| 333 | func (c *Client) fetchDataSource(ctx context.Context, ref Ref) (*Content, error) { |
| 334 | sources := []string{ref.DataSourceID} |
| 335 | if ids, err := c.dataSourcesOf(ctx, ref.DataSourceID); err == nil && len(ids) > 0 { |
| 336 | sources = ids |
| 337 | } else if err != nil { |
| 338 | return nil, err |
| 339 | } |
| 340 | |
| 341 | childPath := append(append([]string{}, ref.Path...), ref.Title) |
| 342 | var kids []Ref |
| 343 | for _, ds := range sources { |
| 344 | err := c.paginate(ctx, "POST", "/v1/data_sources/"+ds+"/query", nil, func(raw json.RawMessage) error { |
| 345 | var o object |
| 346 | if err := json.Unmarshal(raw, &o); err != nil { |
| 347 | return &Error{Code: CodeTraversalFailed} |
| 348 | } |
| 349 | if o.InTrash { |
| 350 | return nil |
| 351 | } |
| 352 | title := titleOf(o) |
| 353 | if title == "" { |
| 354 | // A row with an empty title property. firstLine() over a |
| 355 | // property block would title every such document |
| 356 | // "Status: Shipped", so name it for the collection it is |
| 357 | // in plus its first non-empty value. |
| 358 | title = firstNonEmpty(ref.Title+" row", "Untitled") |
| 359 | if v := firstProperty(o.Properties); v != "" { |
| 360 | title = ref.Title + ": " + v |
| 361 | } |
| 362 | } |
| 363 | kids = append(kids, Ref{ |
| 364 | ID: o.ID, Title: title, Path: childPath, |
| 365 | EditedTime: o.LastEdited, Kind: KindPage, |
| 366 | Properties: o.Properties, |
| 367 | }) |
| 368 | return nil |
| 369 | }) |
| 370 | if err != nil { |
| 371 | return nil, err |
| 372 | } |
| 373 | } |
| 374 | return &Content{Children: kids}, nil |
| 375 | } |
| 376 | |
| 377 | // dataSourcesOf resolves a database id to its data source ids. An id |
| 378 | // that is already a data source answers 404, which is not an error here |
| 379 | // — it means the caller had the right id all along. |
| 380 | func (c *Client) dataSourcesOf(ctx context.Context, databaseID string) ([]string, error) { |
| 381 | var db struct { |
| 382 | DataSources []struct { |
| 383 | ID string `json:"id"` |
| 384 | } `json:"data_sources"` |
| 385 | } |
| 386 | if err := c.get(ctx, "/v1/databases/"+databaseID, &db); err != nil { |
| 387 | var e *Error |
| 388 | if ok := asErr(err, &e); ok && e.Status == 404 { |
| 389 | return nil, nil |
| 390 | } |
| 391 | return nil, err |
| 392 | } |
| 393 | ids := make([]string, 0, len(db.DataSources)) |
| 394 | for _, d := range db.DataSources { |
| 395 | ids = append(ids, d.ID) |
| 396 | } |
| 397 | return ids, nil |
| 398 | } |
| 399 | |
| 400 | // blocks reads every block of a page, recursing into nested blocks but |
| 401 | // NOT into child pages — those are their own documents, and inlining |
| 402 | // them would duplicate every passage into all of its ancestors. |
| 403 | func (c *Client) blocks(ctx context.Context, id string) ([]block, error) { |
| 404 | var out []block |
| 405 | err := c.paginate(ctx, "GET", "/v1/blocks/"+id+"/children", nil, func(raw json.RawMessage) error { |
| 406 | var b block |
| 407 | if err := json.Unmarshal(raw, &b); err != nil { |
| 408 | return &Error{Code: CodeTraversalFailed} |
| 409 | } |
| 410 | b.raw = raw |
| 411 | if b.HasChildren && b.Type != "child_page" && b.Type != "child_database" { |
| 412 | kids, err := c.blocks(ctx, b.ID) |
| 413 | if err != nil { |
| 414 | return err |
| 415 | } |
| 416 | b.children = kids |
| 417 | } |
| 418 | out = append(out, b) |
| 419 | return nil |
| 420 | }) |
| 421 | if err != nil { |
| 422 | return nil, err |
| 423 | } |
| 424 | return out, nil |
| 425 | } |
| 426 | |
amadan