| 1 | import Foundation |
| 2 | import Markdown |
| 3 | |
| 4 | // The native twin of web/static/markdown.js (issue #531): swift-markdown |
| 5 | // (cmark-gfm) parses the message, and this walk normalizes the Markup tree to |
| 6 | // the SAME allowlisted semantic tree the web produces — pinned, node for node, |
| 7 | // by Fixtures/markdown-vector.json (MarkdownVectorTests). When the twins |
| 8 | // disagree, fix the implementations, never the vectors. |
| 9 | // |
| 10 | // Allowlist: emphasis/strong, inline code, links (http(s) only), lists, |
| 11 | // blockquotes, fenced code, strike (single or double tilde), breaks, escapes, |
| 12 | // entities. Everything else renders as literal text. cmark can't *disable* |
| 13 | // constructs the way markdown-it's zero preset can, so disallowed nodes are |
| 14 | // converted back to literal text here — mostly via their source ranges, so the |
| 15 | // member sees exactly what was typed. |
| 16 | // |
| 17 | // Two cmark-side traps this file guards (both vector-pinned): |
| 18 | // - `.disableSmartOpts` is REQUIRED: cmark curls quotes/dashes by default. |
| 19 | // - The strikethrough/tasklist/table extensions are always attached; their |
| 20 | // nodes must fold back to the web-identical shape. |
| 21 | |
| 22 | // MARK: - The tree (the vector contract) |
| 23 | |
| 24 | public enum MDInline: Equatable, Hashable { |
| 25 | case text(String) |
| 26 | case br |
| 27 | case code(String) |
| 28 | case link(href: String, children: [MDInline]) |
| 29 | case strong([MDInline]) |
| 30 | case em([MDInline]) |
| 31 | case strike([MDInline]) |
| 32 | } |
| 33 | |
| 34 | public enum MDBlock: Equatable, Hashable { |
| 35 | case p([MDInline]) |
| 36 | case quote([MDBlock]) |
| 37 | case ul(items: [[MDBlock]]) |
| 38 | case ol(start: Int, items: [[MDBlock]]) |
| 39 | case codeBlock(text: String, lang: String) |
| 40 | } |
| 41 | |
| 42 | // MARK: - Plain projection |
| 43 | |
| 44 | // Markdown-aware PLAIN projection for one-line surfaces (the sidebar's |
| 45 | // last-message preview, notification bodies): escapes resolve (\* → *) and |
| 46 | // the marks themselves drop — the same characters the bubbles show, minus |
| 47 | // the styling. Never raw message text on those surfaces: the composer's |
| 48 | // opt-out escapes leak as visible backslashes (Paul's report, 2026-07-15). |
| 49 | // Twin: plainText (web/static/markdown.js); the `plain` fields in the |
| 50 | // markdown vectors pin the pair. |
| 51 | public func markdownPlainText(_ text: String) -> String { |
| 52 | func inline(_ children: [MDInline]) -> String { |
| 53 | children.map { node in |
| 54 | switch node { |
| 55 | case .text(let s), .code(let s): return s |
| 56 | case .br: return " " |
| 57 | case .link(_, let c), .strong(let c), .em(let c), .strike(let c): return inline(c) |
| 58 | } |
| 59 | }.joined() |
| 60 | } |
| 61 | func blocks(_ list: [MDBlock]) -> String { |
| 62 | list.map { block in |
| 63 | switch block { |
| 64 | case .p(let c): return inline(c) |
| 65 | case .quote(let c): return blocks(c) |
| 66 | case .ul(let items): return items.map(blocks).joined(separator: " ") |
| 67 | case .ol(_, let items): return items.map(blocks).joined(separator: " ") |
| 68 | case .codeBlock(let text, _): return text |
| 69 | } |
| 70 | }.joined(separator: " ") |
| 71 | } |
| 72 | return blocks(markdownTree(text)) |
| 73 | } |
| 74 | |
| 75 | // MARK: - Parse + walk |
| 76 | |
| 77 | public func markdownTree(_ text: String) -> [MDBlock] { |
| 78 | let doc = Document(parsing: text, options: [.disableSmartOpts]) |
| 79 | let walker = MDTreeWalker(source: text) |
| 80 | return walker.blocks(of: doc) |
| 81 | } |
| 82 | |
| 83 | private struct MDTreeWalker { |
| 84 | let source: String |
| 85 | let lines: [Substring] |
| 86 | |
| 87 | init(source: String) { |
| 88 | self.source = source |
| 89 | self.lines = source.split(separator: "\n", omittingEmptySubsequences: false) |
| 90 | } |
| 91 | |
| 92 | // Whole source lines covered by a node — the same recovery web uses for |
| 93 | // unknown block tokens (token.map). 1-based, inclusive. |
| 94 | func sliceLines(_ range: SourceRange?) -> [String] { |
| 95 | guard let range = range else { return [] } |
| 96 | let lo = max(range.lowerBound.line, 1) - 1 |
| 97 | let hi = min(range.upperBound.line, lines.count) - 1 |
| 98 | guard lo <= hi else { return [] } |
| 99 | return lines[lo...hi].map(String.init) |
| 100 | } |
| 101 | |
| 102 | // Column-accurate slice for inline fallbacks (cmark columns count UTF-8 |
| 103 | // bytes; upperBound is exclusive in swift-markdown's Range). |
| 104 | func exactSlice(_ range: SourceRange?) -> String? { |
| 105 | guard let range = range else { return nil } |
| 106 | let loLine = range.lowerBound.line - 1 |
| 107 | let hiLine = range.upperBound.line - 1 |
| 108 | guard loLine >= 0, hiLine < lines.count, loLine <= hiLine else { return nil } |
| 109 | func clip(_ line: Substring, from: Int, to: Int) -> String? { |
| 110 | let bytes = Array(line.utf8) |
| 111 | let lo = max(from, 0), hi = min(to, bytes.count) |
| 112 | guard lo <= hi else { return nil } |
| 113 | return String(decoding: bytes[lo..<hi], as: UTF8.self) |
| 114 | } |
| 115 | if loLine == hiLine { |
| 116 | return clip(lines[loLine], from: range.lowerBound.column - 1, to: range.upperBound.column - 1) |
| 117 | } |
| 118 | var parts: [String] = [] |
| 119 | guard let first = clip(lines[loLine], from: range.lowerBound.column - 1, to: Array(lines[loLine].utf8).count) else { return nil } |
| 120 | parts.append(first) |
| 121 | for i in (loLine + 1)..<hiLine { parts.append(String(lines[i])) } |
| 122 | guard let last = clip(lines[hiLine], from: 0, to: range.upperBound.column - 1) else { return nil } |
| 123 | parts.append(last) |
| 124 | return parts.joined(separator: "\n") |
| 125 | } |
| 126 | |
| 127 | // MARK: blocks |
| 128 | |
| 129 | func blocks(of markup: Markup) -> [MDBlock] { |
| 130 | var out: [MDBlock] = [] |
| 131 | for child in markup.children { |
| 132 | out.append(contentsOf: block(child)) |
| 133 | } |
| 134 | return out |
| 135 | } |
| 136 | |
| 137 | private func block(_ node: Markup) -> [MDBlock] { |
| 138 | switch node { |
| 139 | case let p as Paragraph: |
| 140 | return [.p(inlines(of: p))] |
| 141 | case let q as BlockQuote: |
| 142 | return [.quote(blocks(of: q))] |
| 143 | case let list as UnorderedList: |
| 144 | return [.ul(items: list.children.map { listItem($0) })] |
| 145 | case let list as OrderedList: |
| 146 | return [.ol(start: Int(list.startIndex), items: list.children.map { listItem($0) })] |
| 147 | case let code as CodeBlock: |
| 148 | return [codeBlockOrIndented(code)] |
| 149 | case let h as Heading: |
| 150 | return [heading(h)] |
| 151 | case is ThematicBreak: |
| 152 | // "---" and friends: the exact marker line, literal. |
| 153 | let line = sliceLines(node.range).first ?? "---" |
| 154 | return [.p([.text(line)])] |
| 155 | case let html as HTMLBlock: |
| 156 | // Accepted divergence from web (which never forms HTML blocks and |
| 157 | // still inline-formats between tags): the raw lines, verbatim. |
| 158 | let raw = html.rawHTML.hasSuffix("\n") ? String(html.rawHTML.dropLast()) : html.rawHTML |
| 159 | var children: [MDInline] = [] |
| 160 | for (i, line) in raw.split(separator: "\n", omittingEmptySubsequences: false).enumerated() { |
| 161 | if i > 0 { children.append(.br) } |
| 162 | children.append(.text(String(line))) |
| 163 | } |
| 164 | return [.p(children)] |
| 165 | case is Markdown.Table: |
| 166 | // A lone table row can't re-form a table, so re-parsing line by |
| 167 | // line matches web (pipes literal, inline marks still format). |
| 168 | return [lineByLine(sliceLines(node.range))] |
| 169 | default: |
| 170 | // Unreachable for parsed CommonMark; exact source as a safety net. |
| 171 | let slice = sliceLines(node.range) |
| 172 | return slice.isEmpty ? [] : [lineByLine(slice)] |
| 173 | } |
| 174 | } |
| 175 | |
| 176 | private func listItem(_ node: Markup) -> [MDBlock] { |
| 177 | guard let item = node as? ListItem else { return blocks(of: node) } |
| 178 | var itemBlocks = blocks(of: item) |
| 179 | // cmark-gfm force-parses "- [x] " task items; web keeps the checkbox |
| 180 | // literal. Re-synthesize the prefix into the item's first paragraph. |
| 181 | if let box = item.checkbox { |
| 182 | let prefix = MDInline.text(box == .checked ? "[x] " : "[ ] ") |
| 183 | if case .p(let children) = itemBlocks.first { |
| 184 | itemBlocks[0] = .p(mergeText([prefix] + children)) |
| 185 | } else { |
| 186 | itemBlocks.insert(.p([prefix]), at: 0) |
| 187 | } |
| 188 | } |
| 189 | return itemBlocks |
| 190 | } |
| 191 | |
| 192 | private func codeBlockOrIndented(_ code: CodeBlock) -> MDBlock { |
| 193 | let body = code.code.hasSuffix("\n") ? String(code.code.dropLast()) : code.code |
| 194 | // Fenced vs indented: web never enables the 4-space rule (it mangles |
| 195 | // pasted text), so an indented block folds back to plain lines. cmark |
| 196 | // doesn't flag which form parsed — sniff the opening source line, |
| 197 | // stripping blockquote markers (each allows ≤3 leading spaces) so a |
| 198 | // fence inside a quote still reads as a fence. Whatever spaces remain |
| 199 | // after the final strip are the code indent: a 4th one means indented. |
| 200 | var opener = Substring(sliceLines(code.range).first ?? "") |
| 201 | stripQuotes: while true { |
| 202 | var spaces = 0 |
| 203 | while spaces < 3, opener.first == " " { opener = opener.dropFirst(); spaces += 1 } |
| 204 | guard opener.first == ">" else { break stripQuotes } |
| 205 | opener = opener.dropFirst() |
| 206 | if opener.first == " " { opener = opener.dropFirst() } |
| 207 | } |
| 208 | if opener.hasPrefix("```") || opener.hasPrefix("~~~") { |
| 209 | return .codeBlock(text: body, lang: (code.language ?? "").trimmingCharacters(in: .whitespaces)) |
| 210 | } |
| 211 | var children: [MDInline] = [] |
| 212 | for (i, line) in body.split(separator: "\n", omittingEmptySubsequences: false).enumerated() { |
| 213 | if i > 0 { children.append(.br) } |
| 214 | children.append(.text(String(line))) |
| 215 | } |
| 216 | return .p(children) |
| 217 | } |
| 218 | |
| 219 | private func heading(_ h: Heading) -> MDBlock { |
| 220 | let slice = sliceLines(h.range) |
| 221 | let atx = (slice.first ?? "").drop(while: { $0 == " " }).hasPrefix("#") |
| 222 | if atx { |
| 223 | // Web leaves "# hi **b**" a paragraph whose text keeps the marker |
| 224 | // while inline marks still format — reconstruct the same shape. |
| 225 | let marker = String(repeating: "#", count: h.level) |
| 226 | var children = inlines(of: h) |
| 227 | children.insert(.text(children.isEmpty ? marker : marker + " "), at: 0) |
| 228 | return .p(mergeText(children)) |
| 229 | } |
| 230 | // Setext ("title\n==="): web keeps both lines as a paragraph. |
| 231 | return lineByLine(slice) |
| 232 | } |
| 233 | |
| 234 | // Re-parse each source line alone (none can re-form its multi-line |
| 235 | // construct) and join with breaks — matching how web's disabled block |
| 236 | // rules leave the lines as one paragraph with inline formatting intact. |
| 237 | private func lineByLine(_ slice: [String]) -> MDBlock { |
| 238 | var children: [MDInline] = [] |
| 239 | for (i, line) in slice.enumerated() { |
| 240 | if i > 0 { children.append(.br) } |
| 241 | let doc = Document(parsing: line, options: [.disableSmartOpts]) |
| 242 | let kids = Array(doc.children) |
| 243 | if kids.count == 1, let p = kids.first as? Paragraph { |
| 244 | let walker = MDTreeWalker(source: line) |
| 245 | children.append(contentsOf: walker.inlines(of: p)) |
| 246 | } else if !line.isEmpty { |
| 247 | children.append(.text(line)) |
| 248 | } |
| 249 | } |
| 250 | return .p(mergeText(children)) |
| 251 | } |
| 252 | |
| 253 | // MARK: inlines |
| 254 | |
| 255 | // Plain text accumulates across adjacent literal nodes and is linkified in |
| 256 | // one pass (format.js tokenizeText — the house bare-URL rules), matching |
| 257 | // web's merge-then-linkify order. Inside an explicit link, text stays |
| 258 | // plain: links never nest. |
| 259 | private struct InlineSink { |
| 260 | var out: [MDInline] = [] |
| 261 | var pending = "" |
| 262 | let linkify: Bool |
| 263 | |
| 264 | mutating func flush() { |
| 265 | guard !pending.isEmpty else { return } |
| 266 | if linkify { |
| 267 | for tok in tokenizeText(pending) { |
| 268 | if let href = tok.href { |
| 269 | out.append(.link(href: href.absoluteString, children: [.text(tok.text)])) |
| 270 | } else { |
| 271 | appendText(tok.text) |
| 272 | } |
| 273 | } |
| 274 | } else { |
| 275 | appendText(pending) |
| 276 | } |
| 277 | pending = "" |
| 278 | } |
| 279 | |
| 280 | mutating func appendText(_ s: String) { |
| 281 | guard !s.isEmpty else { return } |
| 282 | if case .text(let prev) = out.last { |
| 283 | out[out.count - 1] = .text(prev + s) |
| 284 | } else { |
| 285 | out.append(.text(s)) |
| 286 | } |
| 287 | } |
| 288 | |
| 289 | mutating func push(_ node: MDInline) { |
| 290 | flush() |
| 291 | out.append(node) |
| 292 | } |
| 293 | } |
| 294 | |
| 295 | func inlines(of markup: Markup, linkify: Bool = true) -> [MDInline] { |
| 296 | var sink = InlineSink(linkify: linkify) |
| 297 | for child in markup.children { |
| 298 | inline(child, into: &sink) |
| 299 | } |
| 300 | sink.flush() |
| 301 | return sink.out |
| 302 | } |
| 303 | |
| 304 | private func inline(_ node: Markup, into sink: inout InlineSink) { |
| 305 | switch node { |
| 306 | case let t as Markdown.Text: |
| 307 | sink.pending += t.string |
| 308 | case is SoftBreak, is LineBreak: |
| 309 | sink.push(.br) |
| 310 | case let c as InlineCode: |
| 311 | sink.push(.code(c.code)) |
| 312 | case let e as Emphasis: |
| 313 | sink.push(.em(inlines(of: e, linkify: sink.linkify))) |
| 314 | case let s as Strong: |
| 315 | sink.push(.strong(inlines(of: s, linkify: sink.linkify))) |
| 316 | case let s as Strikethrough: |
| 317 | sink.push(.strike(inlines(of: s, linkify: sink.linkify))) |
| 318 | case let l as Markdown.Link: |
| 319 | let dest = l.destination ?? "" |
| 320 | if dest.lowercased().hasPrefix("http://") || dest.lowercased().hasPrefix("https://") { |
| 321 | sink.push(.link(href: dest, children: inlines(of: l, linkify: false))) |
| 322 | } else { |
| 323 | // Web's validateLink rejected it before it ever tokenized, so |
| 324 | // the source text stays literal (and linkifies as plain text). |
| 325 | sink.pending += exactSlice(l.range) ?? "[\(plainText(of: l))](\(dest))" |
| 326 | } |
| 327 | case let img as Markdown.Image: |
| 328 | // Images are outside the allowlist on both twins. A valid-scheme |
| 329 | // image was a real token on web (reconstructed, never linkified); |
| 330 | // an invalid one never parsed there, so it stays linkifiable text. |
| 331 | let src = img.source ?? "" |
| 332 | let literal = exactSlice(img.range) ?? ")" |
| 333 | if src.lowercased().hasPrefix("http://") || src.lowercased().hasPrefix("https://") { |
| 334 | sink.flush() |
| 335 | sink.appendText(literal) |
| 336 | } else { |
| 337 | sink.pending += literal |
| 338 | } |
| 339 | case let html as InlineHTML: |
| 340 | // Web never parses inline HTML — the tag is plain text in the same |
| 341 | // run as its neighbours (and linkifies with them). |
| 342 | sink.pending += html.rawHTML |
| 343 | default: |
| 344 | sink.pending += exactSlice(node.range) ?? plainText(of: node) |
| 345 | } |
| 346 | } |
| 347 | |
| 348 | private func plainText(of markup: Markup) -> String { |
| 349 | var out = "" |
| 350 | for child in markup.children { |
| 351 | if let t = child as? Markdown.Text { out += t.string } else { out += plainText(of: child) } |
| 352 | } |
| 353 | return out |
| 354 | } |
| 355 | } |
| 356 | |
| 357 | private func mergeText(_ children: [MDInline]) -> [MDInline] { |
| 358 | var out: [MDInline] = [] |
| 359 | for node in children { |
| 360 | if case .text(let s) = node, case .text(let prev)? = out.last { |
| 361 | out[out.count - 1] = .text(prev + s) |
| 362 | } else { |
| 363 | out.append(node) |
| 364 | } |
| 365 | } |
| 366 | return out |
| 367 | } |
| 368 | |
| 369 | // MARK: - Codable (mirrors the vector JSON exactly) |
| 370 | |
| 371 | extension MDInline: Codable { |
| 372 | private enum CodingKeys: String, CodingKey { case t, text, href, children } |
| 373 | |
| 374 | public init(from decoder: Decoder) throws { |
| 375 | let c = try decoder.container(keyedBy: CodingKeys.self) |
| 376 | switch try c.decode(String.self, forKey: .t) { |
| 377 | case "text": self = .text(try c.decode(String.self, forKey: .text)) |
| 378 | case "br": self = .br |
| 379 | case "code": self = .code(try c.decode(String.self, forKey: .text)) |
| 380 | case "link": self = .link(href: try c.decode(String.self, forKey: .href), |
| 381 | children: try c.decode([MDInline].self, forKey: .children)) |
| 382 | case "strong": self = .strong(try c.decode([MDInline].self, forKey: .children)) |
| 383 | case "em": self = .em(try c.decode([MDInline].self, forKey: .children)) |
| 384 | case "strike": self = .strike(try c.decode([MDInline].self, forKey: .children)) |
| 385 | case let t: throw DecodingError.dataCorruptedError(forKey: .t, in: c, debugDescription: "unknown inline \(t)") |
| 386 | } |
| 387 | } |
| 388 | |
| 389 | public func encode(to encoder: Encoder) throws { |
| 390 | var c = encoder.container(keyedBy: CodingKeys.self) |
| 391 | switch self { |
| 392 | case .text(let s): try c.encode("text", forKey: .t); try c.encode(s, forKey: .text) |
| 393 | case .br: try c.encode("br", forKey: .t) |
| 394 | case .code(let s): try c.encode("code", forKey: .t); try c.encode(s, forKey: .text) |
| 395 | case .link(let href, let children): |
| 396 | try c.encode("link", forKey: .t); try c.encode(href, forKey: .href); try c.encode(children, forKey: .children) |
| 397 | case .strong(let children): try c.encode("strong", forKey: .t); try c.encode(children, forKey: .children) |
| 398 | case .em(let children): try c.encode("em", forKey: .t); try c.encode(children, forKey: .children) |
| 399 | case .strike(let children): try c.encode("strike", forKey: .t); try c.encode(children, forKey: .children) |
| 400 | } |
| 401 | } |
| 402 | } |
| 403 | |
| 404 | extension MDBlock: Codable { |
| 405 | private enum CodingKeys: String, CodingKey { case t, children, items, start, text, lang } |
| 406 | |
| 407 | public init(from decoder: Decoder) throws { |
| 408 | let c = try decoder.container(keyedBy: CodingKeys.self) |
| 409 | switch try c.decode(String.self, forKey: .t) { |
| 410 | case "p": self = .p(try c.decode([MDInline].self, forKey: .children)) |
| 411 | case "quote": self = .quote(try c.decode([MDBlock].self, forKey: .children)) |
| 412 | case "ul": self = .ul(items: try c.decode([[MDBlock]].self, forKey: .items)) |
| 413 | case "ol": self = .ol(start: try c.decode(Int.self, forKey: .start), |
| 414 | items: try c.decode([[MDBlock]].self, forKey: .items)) |
| 415 | case "codeblock": self = .codeBlock(text: try c.decode(String.self, forKey: .text), |
| 416 | lang: try c.decode(String.self, forKey: .lang)) |
| 417 | case let t: throw DecodingError.dataCorruptedError(forKey: .t, in: c, debugDescription: "unknown block \(t)") |
| 418 | } |
| 419 | } |
| 420 | |
| 421 | public func encode(to encoder: Encoder) throws { |
| 422 | var c = encoder.container(keyedBy: CodingKeys.self) |
| 423 | switch self { |
| 424 | case .p(let children): try c.encode("p", forKey: .t); try c.encode(children, forKey: .children) |
| 425 | case .quote(let children): try c.encode("quote", forKey: .t); try c.encode(children, forKey: .children) |
| 426 | case .ul(let items): try c.encode("ul", forKey: .t); try c.encode(items, forKey: .items) |
| 427 | case .ol(let start, let items): |
| 428 | try c.encode("ol", forKey: .t); try c.encode(start, forKey: .start); try c.encode(items, forKey: .items) |
| 429 | case .codeBlock(let text, let lang): |
| 430 | try c.encode("codeblock", forKey: .t); try c.encode(text, forKey: .text); try c.encode(lang, forKey: .lang) |
| 431 | } |
| 432 | } |
| 433 | } |
| 434 | |
amadan