amadan

carlos / platform Public

docsite: parse a markdown corpus and its nav

Ported from rastrillo's internal/docsite, minus the docs:ignore markers
that serve its exported-symbol gate. The platform is an application, not
a library, so there is no symbol surface to hold docs against — the CLI
is the surface, and that gate arrives in a later commit.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Paul Campbell pushed by paul@keymail.dev d371f638890f54a44c10ea01567e98a7ffe53b3c parent 7385c63
6 files changed, +461 −0
  • internal/docsite/docsite.go +331 −0
  • internal/docsite/parse_test.go +88 −0
  • internal/docsite/testdata/corpus/getting-started.md +13 −0
  • internal/docsite/testdata/corpus/index.md +4 −0
  • internal/docsite/testdata/corpus/nav.json +16 −0
  • internal/docsite/testdata/corpus/reference/cli.md +9 −0
diff --git a/internal/docsite/docsite.go b/internal/docsite/docsite.go
new file mode 100644
index 000000000..f16ae1e53
--- /dev/null
+++ b/internal/docsite/docsite.go
@@ -0,0 +1,331 @@
+// Package docsite loads the documentation corpus under docs/site and
+// exposes it in the shape the gates need.
+//
+// The corpus is markdown plus one nav.json holding the table of
+// contents. Everything here is parsing: the tests beside this file are
+// the point, and they are what stop the docs and the code drifting
+// apart. Nothing in this package ships in a binary — docs/site is
+// rendered by the website, which vendors a copy. This package exists so
+// CARLOS's own CI can fail a pull request whose docs no longer describe
+// the platform.
+//
+// The markdown parsing is deliberately small and deliberately
+// fence-aware. It is not a markdown implementation and must never grow
+// into one: it finds ATX headings, fenced code blocks and internal
+// links, and it knows that a "#" inside a shell fence is a comment
+// rather than a heading. Anything more belongs to the renderer.
+package docsite
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/fs"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+)
+
+// Heading is one ATX heading, with the anchor the renderer will give
+// it. Anchor equivalence with the website's markdown-it-anchor
+// configuration is pinned from the website side; see Anchor.
+type Heading struct {
+ Level int
+ Text string
+ Anchor string
+}
+
+// Link is one internal /docs link found in a page's source, with the
+// line it sits on so a failure can name it.
+type Link struct {
+ Target string // page slug, e.g. "migrations" or "reference/db"
+ Fragment string // heading anchor, or "" for a whole-page link
+ Line int
+}
+
+// Fence is one fenced code block and the language on its info string.
+type Fence struct {
+ Lang string
+ Body string
+ Line int
+}
+
+// Page is one documentation page: its nav metadata, its source, and
+// everything the gates read out of it.
+type Page struct {
+ Slug string // "migrations", "reference/db"
+ Label string // nav label, usually shorter than Title
+ Blurb string // one line, rendered on the /docs index
+ Title string // the page's own "# " heading
+ Path string // path on disk
+
+ Body string
+ Headings []Heading
+ Links []Link
+ Fences []Fence
+}
+
+// Section is one group in the sidebar.
+type Section struct {
+ Title string
+ Pages []*Page
+}
+
+// Site is the whole corpus, in reading order.
+type Site struct {
+ // Index is docs/site/index.md, the /docs overview. It sits outside
+ // Sections because it is the page the sidebar's sections are listed
+ // on: an entry for itself would render as a card linking to the page
+ // you are already reading.
+ Index *Page
+ Sections []Section
+ BySlug map[string]*Page
+}
+
+// Pages returns every page in reading order, the index first.
+func (s *Site) Pages() []*Page {
+ var out []*Page
+ if s.Index != nil {
+ out = append(out, s.Index)
+ }
+ for _, sec := range s.Sections {
+ out = append(out, sec.Pages...)
+ }
+ return out
+}
+
+// navFile is nav.json's shape.
+type navFile struct {
+ Sections []struct {
+ Title string `json:"title"`
+ Pages []struct {
+ Slug string `json:"slug"`
+ Label string `json:"label"`
+ Blurb string `json:"blurb"`
+ } `json:"pages"`
+ } `json:"sections"`
+}
+
+// Load reads nav.json and every page it names. A page named by nav.json
+// that does not exist is an error here; a page that exists and is not
+// named is not — that asymmetry is deliberate, because the nav test
+// reports it with a better message than a load failure could.
+func Load(root string) (*Site, error) {
+ raw, err := os.ReadFile(filepath.Join(root, "nav.json"))
+ if err != nil {
+ return nil, fmt.Errorf("read nav.json: %w", err)
+ }
+ var nav navFile
+ if err := json.Unmarshal(raw, &nav); err != nil {
+ return nil, fmt.Errorf("parse nav.json: %w", err)
+ }
+
+ site := &Site{BySlug: map[string]*Page{}}
+ index, err := loadPage(root, "index", "Documentation", "")
+ if err != nil {
+ return nil, err
+ }
+ site.Index = index
+ site.BySlug[index.Slug] = index
+
+ for _, sec := range nav.Sections {
+ out := Section{Title: sec.Title}
+ for _, entry := range sec.Pages {
+ p, err := loadPage(root, entry.Slug, entry.Label, entry.Blurb)
+ if err != nil {
+ return nil, err
+ }
+ out.Pages = append(out.Pages, p)
+ site.BySlug[p.Slug] = p
+ }
+ site.Sections = append(site.Sections, out)
+ }
+ return site, nil
+}
+
+// loadPage reads one page off disk and parses it.
+func loadPage(root, slug, label, blurb string) (*Page, error) {
+ p := &Page{
+ Slug: slug,
+ Label: label,
+ Blurb: blurb,
+ Path: filepath.Join(root, filepath.FromSlash(slug)+".md"),
+ }
+ body, err := os.ReadFile(p.Path)
+ if err != nil {
+ return nil, fmt.Errorf("nav.json names %q: %w", slug, err)
+ }
+ p.Body = string(body)
+ parsePage(p)
+ return p, nil
+}
+
+// parsePage fills in everything derived from a page's source.
+func parsePage(p *Page) {
+ lines := strings.Split(p.Body, "\n")
+
+ inFence := false
+ fenceMark := ""
+ var fence Fence
+
+ for i, line := range lines {
+ lineNo := i + 1
+ trimmed := strings.TrimSpace(line)
+
+ // Fence open/close. The marker must match in kind and be at
+ // least as long as the opener, so a ``` inside a ~~~~ block is
+ // content rather than a close.
+ if mark, info, ok := fenceDelim(trimmed); ok {
+ switch {
+ case !inFence:
+ inFence, fenceMark = true, mark
+ fence = Fence{Lang: info, Line: lineNo}
+ continue
+ case sameFence(mark, fenceMark):
+ inFence = false
+ p.Fences = append(p.Fences, fence)
+ continue
+ }
+ }
+ if inFence {
+ fence.Body += line + "\n"
+ continue
+ }
+
+ if level, text, ok := heading(trimmed); ok {
+ h := Heading{Level: level, Text: text, Anchor: Anchor(text)}
+ p.Headings = append(p.Headings, h)
+ if level == 1 && p.Title == "" {
+ p.Title = text
+ }
+ }
+
+ for _, l := range docsLinks(line) {
+ l.Line = lineNo
+ p.Links = append(p.Links, l)
+ }
+ }
+}
+
+// fenceDelim reports whether a trimmed line opens or closes a fence,
+// returning the marker run and the info string.
+func fenceDelim(line string) (mark, info string, ok bool) {
+ var r byte
+ switch {
+ case strings.HasPrefix(line, "```"):
+ r = '`'
+ case strings.HasPrefix(line, "~~~"):
+ r = '~'
+ default:
+ return "", "", false
+ }
+ n := 0
+ for n < len(line) && line[n] == r {
+ n++
+ }
+ return line[:n], strings.TrimSpace(line[n:]), true
+}
+
+// sameFence reports whether a closing marker can close an opener: same
+// character, and at least as long.
+func sameFence(closing, opening string) bool {
+ if closing == "" || opening == "" || closing[0] != opening[0] {
+ return false
+ }
+ return len(closing) >= len(opening)
+}
+
+// heading parses an ATX heading.
+func heading(line string) (level int, text string, ok bool) {
+ n := 0
+ for n < len(line) && line[n] == '#' {
+ n++
+ }
+ if n == 0 || n > 6 || n >= len(line) || line[n] != ' ' {
+ return 0, "", false
+ }
+ return n, strings.TrimSpace(strings.TrimRight(line[n+1:], " #")), true
+}
+
+// docsLinks finds every markdown link whose target starts with /docs.
+func docsLinks(line string) []Link {
+ var out []Link
+ rest := line
+ for {
+ i := strings.Index(rest, "](/docs")
+ if i < 0 {
+ return out
+ }
+ rest = rest[i+2:]
+ end := strings.IndexByte(rest, ')')
+ if end < 0 {
+ return out
+ }
+ out = append(out, parseDocsHref(rest[:end]))
+ rest = rest[end:]
+ }
+}
+
+// parseDocsHref turns "/docs/reference/db#open" into its slug and
+// fragment. A bare "/docs" is the index, whose slug is "index".
+func parseDocsHref(href string) Link {
+ var l Link
+ if i := strings.IndexByte(href, '#'); i >= 0 {
+ l.Fragment, href = href[i+1:], href[:i]
+ }
+ href = strings.TrimSuffix(href, "/")
+ slug := strings.TrimPrefix(href, "/docs")
+ slug = strings.TrimPrefix(slug, "/")
+ if slug == "" {
+ slug = "index"
+ }
+ l.Target = path.Clean(slug)
+ return l
+}
+
+// Anchor is the heading anchor the rendered page will carry.
+//
+// The website configures markdown-it-anchor with a slugify that matches
+// this function, and both sides assert against docs/site/anchors.json.
+// Both are deliberately plain — lowercase alphanumerics, everything else
+// a separator — because markdown-it-anchor's own default URI-encodes
+// punctuation, which would anchor a heading naming `carlos ship -app`
+// differently on each side.
+func Anchor(text string) string {
+ var b strings.Builder
+ pendingSep := false
+ for _, r := range strings.ToLower(text) {
+ switch {
+ case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
+ if pendingSep && b.Len() > 0 {
+ b.WriteByte('-')
+ }
+ pendingSep = false
+ b.WriteRune(r)
+ default:
+ pendingSep = true
+ }
+ }
+ return b.String()
+}
+
+// MarkdownFiles walks root and returns every page slug on disk, with
+// "reference/" prefixes preserved and ".md" stripped.
+func MarkdownFiles(root string) ([]string, error) {
+ var out []string
+ err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() || !strings.HasSuffix(p, ".md") {
+ return nil
+ }
+ rel, err := filepath.Rel(root, p)
+ if err != nil {
+ return err
+ }
+ out = append(out, strings.TrimSuffix(filepath.ToSlash(rel), ".md"))
+ return nil
+ })
+ return out, err
+}
diff --git a/internal/docsite/parse_test.go b/internal/docsite/parse_test.go
new file mode 100644
index 000000000..b577afe1d
--- /dev/null
+++ b/internal/docsite/parse_test.go
@@ -0,0 +1,88 @@
+package docsite
+
+import "testing"
+
+const testCorpus = "testdata/corpus"
+
+func TestLoadReadsNavAndPages(t *testing.T) {
+ site, err := Load(testCorpus)
+ if err != nil {
+ t.Fatalf("load: %v", err)
+ }
+ if site.Index == nil || site.Index.Title != "Documentation" {
+ t.Fatalf("index not loaded: %+v", site.Index)
+ }
+ if len(site.Sections) != 2 {
+ t.Fatalf("got %d sections, want 2", len(site.Sections))
+ }
+ if got := site.Sections[1].Pages[0].Slug; got != "reference/cli" {
+ t.Errorf("nested slug = %q, want reference/cli", got)
+ }
+ if got := len(site.Pages()); got != 3 {
+ t.Errorf("Pages() = %d, want 3 (index + two)", got)
+ }
+}
+
+func TestParsePageFindsHeadingsLinksAndFences(t *testing.T) {
+ site, err := Load(testCorpus)
+ if err != nil {
+ t.Fatalf("load: %v", err)
+ }
+ p := site.BySlug["getting-started"]
+ if p == nil {
+ t.Fatal("no getting-started page")
+ }
+ if len(p.Headings) != 3 {
+ t.Fatalf("got %d headings, want 3", len(p.Headings))
+ }
+ if p.Headings[1].Anchor != "install-the-cli" {
+ t.Errorf("anchor = %q, want install-the-cli", p.Headings[1].Anchor)
+ }
+ if len(p.Fences) != 1 || p.Fences[0].Lang != "sh" {
+ t.Fatalf("fences = %+v, want one sh fence", p.Fences)
+ }
+ if len(p.Links) != 1 || p.Links[0].Target != "reference/cli" {
+ t.Fatalf("links = %+v, want one to reference/cli", p.Links)
+ }
+}
+
+func TestLinksInsideFencesAreNotLinks(t *testing.T) {
+ site, err := Load(testCorpus)
+ if err != nil {
+ t.Fatalf("load: %v", err)
+ }
+ for _, p := range site.Pages() {
+ for _, l := range p.Links {
+ if l.Target == "" {
+ t.Errorf("%s: empty link target at line %d", p.Slug, l.Line)
+ }
+ }
+ }
+}
+
+func TestIndexLinkResolvesToIndexSlug(t *testing.T) {
+ l := parseDocsHref("/docs")
+ if l.Target != "index" {
+ t.Errorf("parseDocsHref(\"/docs\") target = %q, want index", l.Target)
+ }
+ l = parseDocsHref("/docs/reference/cli#carlos-ship")
+ if l.Target != "reference/cli" || l.Fragment != "carlos-ship" {
+ t.Errorf("got %+v, want reference/cli #carlos-ship", l)
+ }
+}
+
+func TestMarkdownFilesWalksNested(t *testing.T) {
+ files, err := MarkdownFiles(testCorpus)
+ if err != nil {
+ t.Fatalf("walk: %v", err)
+ }
+ want := map[string]bool{"index": true, "getting-started": true, "reference/cli": true}
+ if len(files) != len(want) {
+ t.Fatalf("got %v, want %d files", files, len(want))
+ }
+ for _, f := range files {
+ if !want[f] {
+ t.Errorf("unexpected file %q", f)
+ }
+ }
+}
diff --git a/internal/docsite/testdata/corpus/getting-started.md b/internal/docsite/testdata/corpus/getting-started.md
new file mode 100644
index 000000000..fe9dec4fe
--- /dev/null
+++ b/internal/docsite/testdata/corpus/getting-started.md
@@ -0,0 +1,13 @@
+# Getting started
+
+## Install the CLI
+
+Run it:
+
+```sh
+carlos auth login
+```
+
+## Deploy something
+
+See [the CLI reference](/docs/reference/cli).
diff --git a/internal/docsite/testdata/corpus/index.md b/internal/docsite/testdata/corpus/index.md
new file mode 100644
index 000000000..42c6d985e
--- /dev/null
+++ b/internal/docsite/testdata/corpus/index.md
@@ -0,0 +1,4 @@
+# Documentation
+
+Start with [getting started](/docs/getting-started), or jump to
+[the CLI](/docs/reference/cli#carlos-ship).
diff --git a/internal/docsite/testdata/corpus/nav.json b/internal/docsite/testdata/corpus/nav.json
new file mode 100644
index 000000000..96c4844f1
--- /dev/null
+++ b/internal/docsite/testdata/corpus/nav.json
@@ -0,0 +1,16 @@
+{
+ "sections": [
+ {
+ "title": "Start here",
+ "pages": [
+ { "slug": "getting-started", "label": "Getting started", "blurb": "Install it and deploy something." }
+ ]
+ },
+ {
+ "title": "Reference",
+ "pages": [
+ { "slug": "reference/cli", "label": "The CLI", "blurb": "Every command." }
+ ]
+ }
+ ]
+}
diff --git a/internal/docsite/testdata/corpus/reference/cli.md b/internal/docsite/testdata/corpus/reference/cli.md
new file mode 100644
index 000000000..65756ba51
--- /dev/null
+++ b/internal/docsite/testdata/corpus/reference/cli.md
@@ -0,0 +1,9 @@
+# The CLI
+
+## carlos ship
+
+Publish a release.
+
+```sh
+carlos ship -app hello -version 1.0.0 ./dist
+```