| new file mode 100644 |
| index 0000000..17a8c8c |
| --- /dev/null |
| +++ b/ios/LChatCore/Sources/LChatCore/MarkdownTree.swift |
| @@ -0,0 +1,400 @@ |
| +import Foundation |
| +import Markdown |
| + |
| +// The native twin of web/static/markdown.js (issue #531): swift-markdown |
| +// (cmark-gfm) parses the message, and this walk normalizes the Markup tree to |
| +// the SAME allowlisted semantic tree the web produces — pinned, node for node, |
| +// by Fixtures/markdown-vector.json (MarkdownVectorTests). When the twins |
| +// disagree, fix the implementations, never the vectors. |
| +// |
| +// Allowlist: emphasis/strong, inline code, links (http(s) only), lists, |
| +// blockquotes, fenced code, strike (single or double tilde), breaks, escapes, |
| +// entities. Everything else renders as literal text. cmark can't *disable* |
| +// constructs the way markdown-it's zero preset can, so disallowed nodes are |
| +// converted back to literal text here — mostly via their source ranges, so the |
| +// member sees exactly what was typed. |
| +// |
| +// Two cmark-side traps this file guards (both vector-pinned): |
| +// - `.disableSmartOpts` is REQUIRED: cmark curls quotes/dashes by default. |
| +// - The strikethrough/tasklist/table extensions are always attached; their |
| +// nodes must fold back to the web-identical shape. |
| + |
| +// MARK: - The tree (the vector contract) |
| + |
| +public enum MDInline: Equatable { |
| + case text(String) |
| + case br |
| + case code(String) |
| + case link(href: String, children: [MDInline]) |
| + case strong([MDInline]) |
| + case em([MDInline]) |
| + case strike([MDInline]) |
| +} |
| + |
| +public enum MDBlock: Equatable { |
| + case p([MDInline]) |
| + case quote([MDBlock]) |
| + case ul(items: [[MDBlock]]) |
| + case ol(start: Int, items: [[MDBlock]]) |
| + case codeBlock(text: String, lang: String) |
| +} |
| + |
| +// MARK: - Parse + walk |
| + |
| +public func markdownTree(_ text: String) -> [MDBlock] { |
| + let doc = Document(parsing: text, options: [.disableSmartOpts]) |
| + let walker = MDTreeWalker(source: text) |
| + return walker.blocks(of: doc) |
| +} |
| + |
| +private struct MDTreeWalker { |
| + let source: String |
| + let lines: [Substring] |
| + |
| + init(source: String) { |
| + self.source = source |
| + self.lines = source.split(separator: "\n", omittingEmptySubsequences: false) |
| + } |
| + |
| + // Whole source lines covered by a node — the same recovery web uses for |
| + // unknown block tokens (token.map). 1-based, inclusive. |
| + func sliceLines(_ range: SourceRange?) -> [String] { |
| + guard let range = range else { return [] } |
| + let lo = max(range.lowerBound.line, 1) - 1 |
| + let hi = min(range.upperBound.line, lines.count) - 1 |
| + guard lo <= hi else { return [] } |
| + return lines[lo...hi].map(String.init) |
| + } |
| + |
| + // Column-accurate slice for inline fallbacks (cmark columns count UTF-8 |
| + // bytes; upperBound is exclusive in swift-markdown's Range). |
| + func exactSlice(_ range: SourceRange?) -> String? { |
| + guard let range = range else { return nil } |
| + let loLine = range.lowerBound.line - 1 |
| + let hiLine = range.upperBound.line - 1 |
| + guard loLine >= 0, hiLine < lines.count, loLine <= hiLine else { return nil } |
| + func clip(_ line: Substring, from: Int, to: Int) -> String? { |
| + let bytes = Array(line.utf8) |
| + let lo = max(from, 0), hi = min(to, bytes.count) |
| + guard lo <= hi else { return nil } |
| + return String(decoding: bytes[lo..<hi], as: UTF8.self) |
| + } |
| + if loLine == hiLine { |
| + return clip(lines[loLine], from: range.lowerBound.column - 1, to: range.upperBound.column - 1) |
| + } |
| + var parts: [String] = [] |
| + guard let first = clip(lines[loLine], from: range.lowerBound.column - 1, to: Array(lines[loLine].utf8).count) else { return nil } |
| + parts.append(first) |
| + for i in (loLine + 1)..<hiLine { parts.append(String(lines[i])) } |
| + guard let last = clip(lines[hiLine], from: 0, to: range.upperBound.column - 1) else { return nil } |
| + parts.append(last) |
| + return parts.joined(separator: "\n") |
| + } |
| + |
| + // MARK: blocks |
| + |
| + func blocks(of markup: Markup) -> [MDBlock] { |
| + var out: [MDBlock] = [] |
| + for child in markup.children { |
| + out.append(contentsOf: block(child)) |
| + } |
| + return out |
| + } |
| + |
| + private func block(_ node: Markup) -> [MDBlock] { |
| + switch node { |
| + case let p as Paragraph: |
| + return [.p(inlines(of: p))] |
| + case let q as BlockQuote: |
| + return [.quote(blocks(of: q))] |
| + case let list as UnorderedList: |
| + return [.ul(items: list.children.map { listItem($0) })] |
| + case let list as OrderedList: |
| + return [.ol(start: Int(list.startIndex), items: list.children.map { listItem($0) })] |
| + case let code as CodeBlock: |
| + return [codeBlockOrIndented(code)] |
| + case let h as Heading: |
| + return [heading(h)] |
| + case is ThematicBreak: |
| + // "---" and friends: the exact marker line, literal. |
| + let line = sliceLines(node.range).first ?? "---" |
| + return [.p([.text(line)])] |
| + case let html as HTMLBlock: |
| + // Accepted divergence from web (which never forms HTML blocks and |
| + // still inline-formats between tags): the raw lines, verbatim. |
| + let raw = html.rawHTML.hasSuffix("\n") ? String(html.rawHTML.dropLast()) : html.rawHTML |
| + var children: [MDInline] = [] |
| + for (i, line) in raw.split(separator: "\n", omittingEmptySubsequences: false).enumerated() { |
| + if i > 0 { children.append(.br) } |
| + children.append(.text(String(line))) |
| + } |
| + return [.p(children)] |
| + case is Markdown.Table: |
| + // A lone table row can't re-form a table, so re-parsing line by |
| + // line matches web (pipes literal, inline marks still format). |
| + return [lineByLine(sliceLines(node.range))] |
| + default: |
| + // Unreachable for parsed CommonMark; exact source as a safety net. |
| + let slice = sliceLines(node.range) |
| + return slice.isEmpty ? [] : [lineByLine(slice)] |
| + } |
| + } |
| + |
| + private func listItem(_ node: Markup) -> [MDBlock] { |
| + guard let item = node as? ListItem else { return blocks(of: node) } |
| + var itemBlocks = blocks(of: item) |
| + // cmark-gfm force-parses "- [x] " task items; web keeps the checkbox |
| + // literal. Re-synthesize the prefix into the item's first paragraph. |
| + if let box = item.checkbox { |
| + let prefix = MDInline.text(box == .checked ? "[x] " : "[ ] ") |
| + if case .p(let children) = itemBlocks.first { |
| + itemBlocks[0] = .p(mergeText([prefix] + children)) |
| + } else { |
| + itemBlocks.insert(.p([prefix]), at: 0) |
| + } |
| + } |
| + return itemBlocks |
| + } |
| + |
| + private func codeBlockOrIndented(_ code: CodeBlock) -> MDBlock { |
| + let body = code.code.hasSuffix("\n") ? String(code.code.dropLast()) : code.code |
| + // Fenced vs indented: web never enables the 4-space rule (it mangles |
| + // pasted text), so an indented block folds back to plain lines. cmark |
| + // doesn't flag which form parsed — sniff the opening source line, |
| + // stripping blockquote markers (each allows ≤3 leading spaces) so a |
| + // fence inside a quote still reads as a fence. Whatever spaces remain |
| + // after the final strip are the code indent: a 4th one means indented. |
| + var opener = Substring(sliceLines(code.range).first ?? "") |
| + stripQuotes: while true { |
| + var spaces = 0 |
| + while spaces < 3, opener.first == " " { opener = opener.dropFirst(); spaces += 1 } |
| + guard opener.first == ">" else { break stripQuotes } |
| + opener = opener.dropFirst() |
| + if opener.first == " " { opener = opener.dropFirst() } |
| + } |
| + if opener.hasPrefix("```") || opener.hasPrefix("~~~") { |
| + return .codeBlock(text: body, lang: (code.language ?? "").trimmingCharacters(in: .whitespaces)) |
| + } |
| + var children: [MDInline] = [] |
| + for (i, line) in body.split(separator: "\n", omittingEmptySubsequences: false).enumerated() { |
| + if i > 0 { children.append(.br) } |
| + children.append(.text(String(line))) |
| + } |
| + return .p(children) |
| + } |
| + |
| + private func heading(_ h: Heading) -> MDBlock { |
| + let slice = sliceLines(h.range) |
| + let atx = (slice.first ?? "").drop(while: { $0 == " " }).hasPrefix("#") |
| + if atx { |
| + // Web leaves "# hi **b**" a paragraph whose text keeps the marker |
| + // while inline marks still format — reconstruct the same shape. |
| + let marker = String(repeating: "#", count: h.level) |
| + var children = inlines(of: h) |
| + children.insert(.text(children.isEmpty ? marker : marker + " "), at: 0) |
| + return .p(mergeText(children)) |
| + } |
| + // Setext ("title\n==="): web keeps both lines as a paragraph. |
| + return lineByLine(slice) |
| + } |
| + |
| + // Re-parse each source line alone (none can re-form its multi-line |
| + // construct) and join with breaks — matching how web's disabled block |
| + // rules leave the lines as one paragraph with inline formatting intact. |
| + private func lineByLine(_ slice: [String]) -> MDBlock { |
| + var children: [MDInline] = [] |
| + for (i, line) in slice.enumerated() { |
| + if i > 0 { children.append(.br) } |
| + let doc = Document(parsing: line, options: [.disableSmartOpts]) |
| + let kids = Array(doc.children) |
| + if kids.count == 1, let p = kids.first as? Paragraph { |
| + let walker = MDTreeWalker(source: line) |
| + children.append(contentsOf: walker.inlines(of: p)) |
| + } else if !line.isEmpty { |
| + children.append(.text(line)) |
| + } |
| + } |
| + return .p(mergeText(children)) |
| + } |
| + |
| + // MARK: inlines |
| + |
| + // Plain text accumulates across adjacent literal nodes and is linkified in |
| + // one pass (format.js tokenizeText — the house bare-URL rules), matching |
| + // web's merge-then-linkify order. Inside an explicit link, text stays |
| + // plain: links never nest. |
| + private struct InlineSink { |
| + var out: [MDInline] = [] |
| + var pending = "" |
| + let linkify: Bool |
| + |
| + mutating func flush() { |
| + guard !pending.isEmpty else { return } |
| + if linkify { |
| + for tok in tokenizeText(pending) { |
| + if let href = tok.href { |
| + out.append(.link(href: href.absoluteString, children: [.text(tok.text)])) |
| + } else { |
| + appendText(tok.text) |
| + } |
| + } |
| + } else { |
| + appendText(pending) |
| + } |
| + pending = "" |
| + } |
| + |
| + mutating func appendText(_ s: String) { |
| + guard !s.isEmpty else { return } |
| + if case .text(let prev) = out.last { |
| + out[out.count - 1] = .text(prev + s) |
| + } else { |
| + out.append(.text(s)) |
| + } |
| + } |
| + |
| + mutating func push(_ node: MDInline) { |
| + flush() |
| + out.append(node) |
| + } |
| + } |
| + |
| + func inlines(of markup: Markup, linkify: Bool = true) -> [MDInline] { |
| + var sink = InlineSink(linkify: linkify) |
| + for child in markup.children { |
| + inline(child, into: &sink) |
| + } |
| + sink.flush() |
| + return sink.out |
| + } |
| + |
| + private func inline(_ node: Markup, into sink: inout InlineSink) { |
| + switch node { |
| + case let t as Markdown.Text: |
| + sink.pending += t.string |
| + case is SoftBreak, is LineBreak: |
| + sink.push(.br) |
| + case let c as InlineCode: |
| + sink.push(.code(c.code)) |
| + case let e as Emphasis: |
| + sink.push(.em(inlines(of: e, linkify: sink.linkify))) |
| + case let s as Strong: |
| + sink.push(.strong(inlines(of: s, linkify: sink.linkify))) |
| + case let s as Strikethrough: |
| + sink.push(.strike(inlines(of: s, linkify: sink.linkify))) |
| + case let l as Markdown.Link: |
| + let dest = l.destination ?? "" |
| + if dest.lowercased().hasPrefix("http://") || dest.lowercased().hasPrefix("https://") { |
| + sink.push(.link(href: dest, children: inlines(of: l, linkify: false))) |
| + } else { |
| + // Web's validateLink rejected it before it ever tokenized, so |
| + // the source text stays literal (and linkifies as plain text). |
| + sink.pending += exactSlice(l.range) ?? "[\(plainText(of: l))](\(dest))" |
| + } |
| + case let img as Markdown.Image: |
| + // Images are outside the allowlist on both twins. A valid-scheme |
| + // image was a real token on web (reconstructed, never linkified); |
| + // an invalid one never parsed there, so it stays linkifiable text. |
| + let src = img.source ?? "" |
| + let literal = exactSlice(img.range) ?? ")" |
| + if src.lowercased().hasPrefix("http://") || src.lowercased().hasPrefix("https://") { |
| + sink.flush() |
| + sink.appendText(literal) |
| + } else { |
| + sink.pending += literal |
| + } |
| + case let html as InlineHTML: |
| + // Web never parses inline HTML — the tag is plain text in the same |
| + // run as its neighbours (and linkifies with them). |
| + sink.pending += html.rawHTML |
| + default: |
| + sink.pending += exactSlice(node.range) ?? plainText(of: node) |
| + } |
| + } |
| + |
| + private func plainText(of markup: Markup) -> String { |
| + var out = "" |
| + for child in markup.children { |
| + if let t = child as? Markdown.Text { out += t.string } else { out += plainText(of: child) } |
| + } |
| + return out |
| + } |
| +} |
| + |
| +private func mergeText(_ children: [MDInline]) -> [MDInline] { |
| + var out: [MDInline] = [] |
| + for node in children { |
| + if case .text(let s) = node, case .text(let prev)? = out.last { |
| + out[out.count - 1] = .text(prev + s) |
| + } else { |
| + out.append(node) |
| + } |
| + } |
| + return out |
| +} |
| + |
| +// MARK: - Codable (mirrors the vector JSON exactly) |
| + |
| +extension MDInline: Codable { |
| + private enum CodingKeys: String, CodingKey { case t, text, href, children } |
| + |
| + public init(from decoder: Decoder) throws { |
| + let c = try decoder.container(keyedBy: CodingKeys.self) |
| + switch try c.decode(String.self, forKey: .t) { |
| + case "text": self = .text(try c.decode(String.self, forKey: .text)) |
| + case "br": self = .br |
| + case "code": self = .code(try c.decode(String.self, forKey: .text)) |
| + case "link": self = .link(href: try c.decode(String.self, forKey: .href), |
| + children: try c.decode([MDInline].self, forKey: .children)) |
| + case "strong": self = .strong(try c.decode([MDInline].self, forKey: .children)) |
| + case "em": self = .em(try c.decode([MDInline].self, forKey: .children)) |
| + case "strike": self = .strike(try c.decode([MDInline].self, forKey: .children)) |
| + case let t: throw DecodingError.dataCorruptedError(forKey: .t, in: c, debugDescription: "unknown inline \(t)") |
| + } |
| + } |
| + |
| + public func encode(to encoder: Encoder) throws { |
| + var c = encoder.container(keyedBy: CodingKeys.self) |
| + switch self { |
| + case .text(let s): try c.encode("text", forKey: .t); try c.encode(s, forKey: .text) |
| + case .br: try c.encode("br", forKey: .t) |
| + case .code(let s): try c.encode("code", forKey: .t); try c.encode(s, forKey: .text) |
| + case .link(let href, let children): |
| + try c.encode("link", forKey: .t); try c.encode(href, forKey: .href); try c.encode(children, forKey: .children) |
| + case .strong(let children): try c.encode("strong", forKey: .t); try c.encode(children, forKey: .children) |
| + case .em(let children): try c.encode("em", forKey: .t); try c.encode(children, forKey: .children) |
| + case .strike(let children): try c.encode("strike", forKey: .t); try c.encode(children, forKey: .children) |
| + } |
| + } |
| +} |
| + |
| +extension MDBlock: Codable { |
| + private enum CodingKeys: String, CodingKey { case t, children, items, start, text, lang } |
| + |
| + public init(from decoder: Decoder) throws { |
| + let c = try decoder.container(keyedBy: CodingKeys.self) |
| + switch try c.decode(String.self, forKey: .t) { |
| + case "p": self = .p(try c.decode([MDInline].self, forKey: .children)) |
| + case "quote": self = .quote(try c.decode([MDBlock].self, forKey: .children)) |
| + case "ul": self = .ul(items: try c.decode([[MDBlock]].self, forKey: .items)) |
| + case "ol": self = .ol(start: try c.decode(Int.self, forKey: .start), |
| + items: try c.decode([[MDBlock]].self, forKey: .items)) |
| + case "codeblock": self = .codeBlock(text: try c.decode(String.self, forKey: .text), |
| + lang: try c.decode(String.self, forKey: .lang)) |
| + case let t: throw DecodingError.dataCorruptedError(forKey: .t, in: c, debugDescription: "unknown block \(t)") |
| + } |
| + } |
| + |
| + public func encode(to encoder: Encoder) throws { |
| + var c = encoder.container(keyedBy: CodingKeys.self) |
| + switch self { |
| + case .p(let children): try c.encode("p", forKey: .t); try c.encode(children, forKey: .children) |
| + case .quote(let children): try c.encode("quote", forKey: .t); try c.encode(children, forKey: .children) |
| + case .ul(let items): try c.encode("ul", forKey: .t); try c.encode(items, forKey: .items) |
| + case .ol(let start, let items): |
| + try c.encode("ol", forKey: .t); try c.encode(start, forKey: .start); try c.encode(items, forKey: .items) |
| + case .codeBlock(let text, let lang): |
| + try c.encode("codeblock", forKey: .t); try c.encode(text, forKey: .text); try c.encode(lang, forKey: .lang) |
| + } |
| + } |
| +} |
amadan