| 1 | // Package docsite loads the documentation corpus under docs/site and |
| 2 | // exposes it in the shape the gates need. |
| 3 | // |
| 4 | // The corpus is markdown plus one nav.json holding the table of |
| 5 | // contents. Everything here is parsing: the tests beside this file are |
| 6 | // the point, and they are what stop the docs and the code drifting |
| 7 | // apart. Nothing in this package ships in a binary — docs/site is |
| 8 | // rendered by the website, which vendors a copy. This package exists so |
| 9 | // CARLOS's own CI can fail a pull request whose docs no longer describe |
| 10 | // the platform. |
| 11 | // |
| 12 | // The markdown parsing is deliberately small and deliberately |
| 13 | // fence-aware. It is not a markdown implementation and must never grow |
| 14 | // into one: it finds ATX headings, fenced code blocks and internal |
| 15 | // links, and it knows that a "#" inside a shell fence is a comment |
| 16 | // rather than a heading. Anything more belongs to the renderer. |
| 17 | package docsite |
| 18 | |
| 19 | import ( |
| 20 | "encoding/json" |
| 21 | "fmt" |
| 22 | "io/fs" |
| 23 | "os" |
| 24 | "path" |
| 25 | "path/filepath" |
| 26 | "strings" |
| 27 | ) |
| 28 | |
| 29 | // Heading is one ATX heading, with the anchor the renderer will give |
| 30 | // it. Anchor equivalence with the website's markdown-it-anchor |
| 31 | // configuration is pinned from the website side; see Anchor. |
| 32 | type Heading struct { |
| 33 | Level int |
| 34 | Text string |
| 35 | Anchor string |
| 36 | Line int // 1-based line in the page source |
| 37 | } |
| 38 | |
| 39 | // Link is one internal /docs link found in a page's source, with the |
| 40 | // line it sits on so a failure can name it. |
| 41 | type Link struct { |
| 42 | Target string // page slug, e.g. "migrations" or "reference/db" |
| 43 | Fragment string // heading anchor, or "" for a whole-page link |
| 44 | Line int |
| 45 | } |
| 46 | |
| 47 | // Fence is one fenced code block and the language on its info string. |
| 48 | type Fence struct { |
| 49 | Lang string |
| 50 | Body string |
| 51 | Line int |
| 52 | } |
| 53 | |
| 54 | // Page is one documentation page: its nav metadata, its source, and |
| 55 | // everything the gates read out of it. |
| 56 | type Page struct { |
| 57 | Slug string // "migrations", "reference/db" |
| 58 | Label string // nav label, usually shorter than Title |
| 59 | Blurb string // one line, rendered on the /docs index |
| 60 | Title string // the page's own "# " heading |
| 61 | Path string // path on disk |
| 62 | |
| 63 | Body string |
| 64 | Headings []Heading |
| 65 | Links []Link |
| 66 | Fences []Fence |
| 67 | } |
| 68 | |
| 69 | // Section is one group in the sidebar. |
| 70 | type Section struct { |
| 71 | Title string |
| 72 | Pages []*Page |
| 73 | } |
| 74 | |
| 75 | // Site is the whole corpus, in reading order. |
| 76 | type Site struct { |
| 77 | // Index is docs/site/index.md, the /docs overview. It sits outside |
| 78 | // Sections because it is the page the sidebar's sections are listed |
| 79 | // on: an entry for itself would render as a card linking to the page |
| 80 | // you are already reading. |
| 81 | Index *Page |
| 82 | Sections []Section |
| 83 | BySlug map[string]*Page |
| 84 | } |
| 85 | |
| 86 | // Pages returns every page in reading order, the index first. |
| 87 | func (s *Site) Pages() []*Page { |
| 88 | var out []*Page |
| 89 | if s.Index != nil { |
| 90 | out = append(out, s.Index) |
| 91 | } |
| 92 | for _, sec := range s.Sections { |
| 93 | out = append(out, sec.Pages...) |
| 94 | } |
| 95 | return out |
| 96 | } |
| 97 | |
| 98 | // navFile is nav.json's shape. |
| 99 | type navFile struct { |
| 100 | Sections []struct { |
| 101 | Title string `json:"title"` |
| 102 | Pages []struct { |
| 103 | Slug string `json:"slug"` |
| 104 | Label string `json:"label"` |
| 105 | Blurb string `json:"blurb"` |
| 106 | } `json:"pages"` |
| 107 | } `json:"sections"` |
| 108 | } |
| 109 | |
| 110 | // Load reads nav.json and every page it names. A page named by nav.json |
| 111 | // that does not exist is an error here; a page that exists and is not |
| 112 | // named is not — that asymmetry is deliberate, because the nav test |
| 113 | // reports it with a better message than a load failure could. |
| 114 | func Load(root string) (*Site, error) { |
| 115 | raw, err := os.ReadFile(filepath.Join(root, "nav.json")) |
| 116 | if err != nil { |
| 117 | return nil, fmt.Errorf("read nav.json: %w", err) |
| 118 | } |
| 119 | var nav navFile |
| 120 | if err := json.Unmarshal(raw, &nav); err != nil { |
| 121 | return nil, fmt.Errorf("parse nav.json: %w", err) |
| 122 | } |
| 123 | |
| 124 | site := &Site{BySlug: map[string]*Page{}} |
| 125 | index, err := loadPage(root, "index", "Documentation", "") |
| 126 | if err != nil { |
| 127 | return nil, err |
| 128 | } |
| 129 | site.Index = index |
| 130 | site.BySlug[index.Slug] = index |
| 131 | |
| 132 | for _, sec := range nav.Sections { |
| 133 | out := Section{Title: sec.Title} |
| 134 | for _, entry := range sec.Pages { |
| 135 | p, err := loadPage(root, entry.Slug, entry.Label, entry.Blurb) |
| 136 | if err != nil { |
| 137 | return nil, err |
| 138 | } |
| 139 | out.Pages = append(out.Pages, p) |
| 140 | site.BySlug[p.Slug] = p |
| 141 | } |
| 142 | site.Sections = append(site.Sections, out) |
| 143 | } |
| 144 | return site, nil |
| 145 | } |
| 146 | |
| 147 | // loadPage reads one page off disk and parses it. |
| 148 | func loadPage(root, slug, label, blurb string) (*Page, error) { |
| 149 | p := &Page{ |
| 150 | Slug: slug, |
| 151 | Label: label, |
| 152 | Blurb: blurb, |
| 153 | Path: filepath.Join(root, filepath.FromSlash(slug)+".md"), |
| 154 | } |
| 155 | body, err := os.ReadFile(p.Path) |
| 156 | if err != nil { |
| 157 | return nil, fmt.Errorf("nav.json names %q: %w", slug, err) |
| 158 | } |
| 159 | p.Body = string(body) |
| 160 | parsePage(p) |
| 161 | return p, nil |
| 162 | } |
| 163 | |
| 164 | // Parse parses a page's source that is not on disk, such as an older |
| 165 | // revision of one. The changelog uses it to find which section a past |
| 166 | // commit changed. |
| 167 | func Parse(slug, body string) *Page { |
| 168 | p := &Page{Slug: slug, Body: body} |
| 169 | parsePage(p) |
| 170 | return p |
| 171 | } |
| 172 | |
| 173 | // parsePage fills in everything derived from a page's source. |
| 174 | func parsePage(p *Page) { |
| 175 | lines := strings.Split(p.Body, "\n") |
| 176 | |
| 177 | inFence := false |
| 178 | fenceMark := "" |
| 179 | var fence Fence |
| 180 | |
| 181 | for i, line := range lines { |
| 182 | lineNo := i + 1 |
| 183 | trimmed := strings.TrimSpace(line) |
| 184 | |
| 185 | // Fence open/close. The marker must match in kind and be at |
| 186 | // least as long as the opener, so a ``` inside a ~~~~ block is |
| 187 | // content rather than a close. |
| 188 | if mark, info, ok := fenceDelim(trimmed); ok { |
| 189 | switch { |
| 190 | case !inFence: |
| 191 | inFence, fenceMark = true, mark |
| 192 | fence = Fence{Lang: info, Line: lineNo} |
| 193 | continue |
| 194 | case sameFence(mark, fenceMark): |
| 195 | inFence = false |
| 196 | p.Fences = append(p.Fences, fence) |
| 197 | continue |
| 198 | } |
| 199 | } |
| 200 | if inFence { |
| 201 | fence.Body += line + "\n" |
| 202 | continue |
| 203 | } |
| 204 | |
| 205 | if level, text, ok := heading(trimmed); ok { |
| 206 | h := Heading{Level: level, Text: text, Anchor: Anchor(text), Line: lineNo} |
| 207 | p.Headings = append(p.Headings, h) |
| 208 | if level == 1 && p.Title == "" { |
| 209 | p.Title = text |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | for _, l := range docsLinks(line) { |
| 214 | l.Line = lineNo |
| 215 | p.Links = append(p.Links, l) |
| 216 | } |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | // fenceDelim reports whether a trimmed line opens or closes a fence, |
| 221 | // returning the marker run and the info string. |
| 222 | func fenceDelim(line string) (mark, info string, ok bool) { |
| 223 | var r byte |
| 224 | switch { |
| 225 | case strings.HasPrefix(line, "```"): |
| 226 | r = '`' |
| 227 | case strings.HasPrefix(line, "~~~"): |
| 228 | r = '~' |
| 229 | default: |
| 230 | return "", "", false |
| 231 | } |
| 232 | n := 0 |
| 233 | for n < len(line) && line[n] == r { |
| 234 | n++ |
| 235 | } |
| 236 | return line[:n], strings.TrimSpace(line[n:]), true |
| 237 | } |
| 238 | |
| 239 | // sameFence reports whether a closing marker can close an opener: same |
| 240 | // character, and at least as long. |
| 241 | func sameFence(closing, opening string) bool { |
| 242 | if closing == "" || opening == "" || closing[0] != opening[0] { |
| 243 | return false |
| 244 | } |
| 245 | return len(closing) >= len(opening) |
| 246 | } |
| 247 | |
| 248 | // heading parses an ATX heading. |
| 249 | func heading(line string) (level int, text string, ok bool) { |
| 250 | n := 0 |
| 251 | for n < len(line) && line[n] == '#' { |
| 252 | n++ |
| 253 | } |
| 254 | if n == 0 || n > 6 || n >= len(line) || line[n] != ' ' { |
| 255 | return 0, "", false |
| 256 | } |
| 257 | return n, strings.TrimSpace(strings.TrimRight(line[n+1:], " #")), true |
| 258 | } |
| 259 | |
| 260 | // docsLinks finds every markdown link whose target starts with /docs. |
| 261 | func docsLinks(line string) []Link { |
| 262 | var out []Link |
| 263 | rest := line |
| 264 | for { |
| 265 | i := strings.Index(rest, "](/docs") |
| 266 | if i < 0 { |
| 267 | return out |
| 268 | } |
| 269 | rest = rest[i+2:] |
| 270 | end := strings.IndexByte(rest, ')') |
| 271 | if end < 0 { |
| 272 | return out |
| 273 | } |
| 274 | out = append(out, parseDocsHref(rest[:end])) |
| 275 | rest = rest[end:] |
| 276 | } |
| 277 | } |
| 278 | |
| 279 | // parseDocsHref turns "/docs/reference/db#open" into its slug and |
| 280 | // fragment. A bare "/docs" is the index, whose slug is "index". |
| 281 | func parseDocsHref(href string) Link { |
| 282 | var l Link |
| 283 | if i := strings.IndexByte(href, '#'); i >= 0 { |
| 284 | l.Fragment, href = href[i+1:], href[:i] |
| 285 | } |
| 286 | href = strings.TrimSuffix(href, "/") |
| 287 | slug := strings.TrimPrefix(href, "/docs") |
| 288 | slug = strings.TrimPrefix(slug, "/") |
| 289 | if slug == "" { |
| 290 | slug = "index" |
| 291 | } |
| 292 | l.Target = path.Clean(slug) |
| 293 | return l |
| 294 | } |
| 295 | |
| 296 | // Anchor is the heading anchor the rendered page will carry. |
| 297 | // |
| 298 | // The website configures markdown-it-anchor with a slugify that matches |
| 299 | // this function, and both sides assert against docs/site/anchors.json. |
| 300 | // Both are deliberately plain — lowercase alphanumerics, everything else |
| 301 | // a separator — because markdown-it-anchor's own default URI-encodes |
| 302 | // punctuation, which would anchor a heading naming `carlos ship -app` |
| 303 | // differently on each side. |
| 304 | func Anchor(text string) string { |
| 305 | var b strings.Builder |
| 306 | pendingSep := false |
| 307 | for _, r := range strings.ToLower(text) { |
| 308 | switch { |
| 309 | case r >= 'a' && r <= 'z', r >= '0' && r <= '9': |
| 310 | if pendingSep && b.Len() > 0 { |
| 311 | b.WriteByte('-') |
| 312 | } |
| 313 | pendingSep = false |
| 314 | b.WriteRune(r) |
| 315 | default: |
| 316 | pendingSep = true |
| 317 | } |
| 318 | } |
| 319 | return b.String() |
| 320 | } |
| 321 | |
| 322 | // MarkdownFiles walks root and returns every page slug on disk, with |
| 323 | // "reference/" prefixes preserved and ".md" stripped. |
| 324 | func MarkdownFiles(root string) ([]string, error) { |
| 325 | var out []string |
| 326 | err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { |
| 327 | if err != nil { |
| 328 | return err |
| 329 | } |
| 330 | if d.IsDir() || !strings.HasSuffix(p, ".md") { |
| 331 | return nil |
| 332 | } |
| 333 | rel, err := filepath.Rel(root, p) |
| 334 | if err != nil { |
| 335 | return err |
| 336 | } |
| 337 | out = append(out, strings.TrimSuffix(filepath.ToSlash(rel), ".md")) |
| 338 | return nil |
| 339 | }) |
| 340 | return out, err |
| 341 | } |
| 342 | |
amadan