diff --git a/packages/ui/HANDOFF.md b/packages/ui/HANDOFF.md index c39e031ed..b2bdf6c9c 100644 --- a/packages/ui/HANDOFF.md +++ b/packages/ui/HANDOFF.md @@ -267,6 +267,8 @@ interface Annotation { **Honesty note:** the failure path (step 4) is exercised in real use but is **not covered by automated tests** — nothing in the suite asserts the stale-anchor behavior. Treat the described degradation as accurate-but-unverified-by-CI, and test it in your integration if you depend on it. +**Migration caveat — reference-style link resolution (#923):** `parseMarkdownToBlocks` now rewrites CommonMark reference links (`[text][id]`) and blanks their `[id]: url` definitions before splitting into blocks, so documents containing that syntax render differently than they did before this pass existed — a `[text][id]` pair that used to render as literal bracket text now renders as a link, and the definition line disappears from the rendered DOM entirely. That changes both the text and the per-tag DOM index at the affected positions. Any annotation whose `startMeta`/`endMeta` was captured against the *old* (pre-resolution) render of such a document — i.e. persisted before a host upgrades past this change — can restore onto the wrong text after upgrading, same as any other DOM-structure change described above; the text-search fallback (step 3) is the recovery path, and `originalText` is what to fall back to if you need to re-anchor server-side. + --- ## Known rough edges (and why they're fine for now) diff --git a/packages/ui/utils/parser.test.ts b/packages/ui/utils/parser.test.ts index f7f148039..245a69519 100644 --- a/packages/ui/utils/parser.test.ts +++ b/packages/ui/utils/parser.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect } from "bun:test"; -import { parseMarkdownToBlocks, computeListIndices, extractFrontmatter, exportAnnotations } from "./parser"; +import { parseMarkdownToBlocks, computeListIndices, extractFrontmatter, exportAnnotations, resolveReferenceLinks } from "./parser"; import { shouldStripFrontmatter } from "@plannotator/core/annotatable"; import type { Block } from "../types"; @@ -15,6 +15,324 @@ const li = (level: number, ordered: boolean, orderedStart?: number): Block => ({ startLine: 1, }); +describe("resolveReferenceLinks (#923)", () => { + test("resolves full, collapsed, and shortcut references to inline links", () => { + expect(resolveReferenceLinks("[text][id]\n\n[id]: https://e.com")).toBe( + "[text](https://e.com)\n\n", + ); + expect(resolveReferenceLinks("[text][]\n\n[text]: https://e.com")).toBe( + "[text](https://e.com)\n\n", + ); + expect(resolveReferenceLinks("[text]\n\n[text]: https://e.com")).toBe( + "[text](https://e.com)\n\n", + ); + }); + + test("matches labels case-insensitively and collapses internal whitespace", () => { + expect(resolveReferenceLinks("[Text][My Ref]\n\n[my ref]: https://e.com")).toBe( + "[Text](https://e.com)\n\n", + ); + }); + + test("supports the angle-bracket destination form and reference images", () => { + expect(resolveReferenceLinks("[x][id]\n\n[id]: ")).toBe( + "[x](https://e.com)\n\n", + ); + expect(resolveReferenceLinks("![alt][id]\n\n[id]: /img.png")).toBe( + "![alt](/img.png)\n\n", + ); + }); + + test("uses the first definition when a label is defined more than once", () => { + expect( + resolveReferenceLinks("[id]: https://one.com\n[id]: https://two.com\n\n[x][id]"), + ).toBe("\n\n\n[x](https://one.com)"); + }); + + test("leaves unknown references and non-reference brackets untouched", () => { + expect(resolveReferenceLinks("[text][missing].")).toBe("[text][missing]."); + // No definitions at all: fast path returns the input unchanged. + expect(resolveReferenceLinks("array [0] and [TODO] here")).toBe( + "array [0] and [TODO] here", + ); + // A shortcut that does not name a definition stays literal even when other + // definitions exist. The definition itself is never consumed by anything + // in this document, so it stays visible too (PR #1168: unused definitions + // are not blanked). + expect(resolveReferenceLinks("[TODO] and [0]\n\n[id]: https://e.com")).toBe( + "[TODO] and [0]\n\n[id]: https://e.com", + ); + }); + + test("does not double-link an inline link whose text matches a definition", () => { + // The inline link uses its own explicit URL and never consumes the + // definition, so the definition stays visible (PR #1168). + expect( + resolveReferenceLinks("[text](https://real.com)\n\n[text]: https://def.com"), + ).toBe("[text](https://real.com)\n\n[text]: https://def.com"); + }); + + test("never rewrites references inside fenced code blocks or inline code spans", () => { + // Both definitions are only ever "referenced" from inside a protected + // region (a fence, an inline code span), so neither reference resolves + // and neither definition is consumed — both stay visible verbatim + // (PR #1168). + expect(resolveReferenceLinks("```\n[a][b]\n```\n\n[b]: https://e.com")).toBe( + "```\n[a][b]\n```\n\n[b]: https://e.com", + ); + expect(resolveReferenceLinks("use `[a][b]` here\n\n[b]: https://e.com")).toBe( + "use `[a][b]` here\n\n[b]: https://e.com", + ); + }); + + test("does not collect a definition that sits inside a fenced code block", () => { + // The only `[id]:` is inside code, so `[id]` outside stays a literal shortcut. + expect( + resolveReferenceLinks("```\n[id]: https://code.com\n```\n\n[id]"), + ).toBe("```\n[id]: https://code.com\n```\n\n[id]"); + }); + + test("does not treat prose with an invalid title as a definition", () => { + expect( + resolveReferenceLinks("[Reminder]: call the bank tomorrow\n\n[Reminder]"), + ).toBe("[Reminder]: call the bank tomorrow\n\n[Reminder]"); + }); + + test("does not corrupt bare space-delimited numbers on a resolved line", () => { + expect(resolveReferenceLinks("value is 0 and 1 and [x][id]\n\n[id]: https://e.com")).toBe( + "value is 0 and 1 and [x](https://e.com)\n\n", + ); + }); + + test("blanks definition lines in place so block start-lines stay accurate", () => { + const blocks = parseMarkdownToBlocks("[id]: https://e.com\n\n# Heading\n\ntext [x][id]"); + // The definition line renders nothing; the heading and paragraph keep their + // original source line numbers. + const heading = blocks.find((b) => b.type === "heading"); + const paragraph = blocks.find((b) => b.type === "paragraph"); + expect(heading?.startLine).toBe(3); + expect(paragraph?.startLine).toBe(5); + expect(paragraph?.content).toBe("text [x](https://e.com)"); + expect(blocks.some((b) => b.content.includes("[id]: https://e.com"))).toBe(false); + }); + + test("does not delete a definition-shaped line that continues a paragraph", () => { + // CommonMark: a definition cannot interrupt a paragraph. The second line + // must survive as content, not be silently blanked. + expect(resolveReferenceLinks("The config keys are:\n[timeout]: 30")).toBe( + "The config keys are:\n[timeout]: 30", + ); + expect(resolveReferenceLinks("text before\n[id]: url\nmore [x][id]")).toBe( + "text before\n[id]: url\nmore [x][id]", + ); + }); + + test("collects a definition after a blank line, a code fence, or another definition", () => { + expect(resolveReferenceLinks("[a]: https://one.com\n[b]: https://two.com\n\n[x][a] [y][b]")).toBe( + "\n\n\n[x](https://one.com) [y](https://two.com)", + ); + expect(resolveReferenceLinks("```\ncode\n```\n[id]: https://e.com\n\n[x][id]")).toBe( + "```\ncode\n```\n\n\n[x](https://e.com)", + ); + }); + + test("does not clobber a checked task-list item when an [x] definition exists", () => { + // The checkbox guard means the task-list `[x]` never resolves as a + // reference, so the "x" definition is never consumed and stays visible + // (PR #1168). + expect(resolveReferenceLinks("- [x] done task\n- [ ] todo\n\n[x]: https://e.com")).toBe( + "- [x] done task\n- [ ] todo\n\n[x]: https://e.com", + ); + expect(resolveReferenceLinks("1. [x] done\n\n[x]: https://e.com")).toBe( + "1. [x] done\n\n[x]: https://e.com", + ); + }); + + test("resolves the shortcut image form", () => { + expect(resolveReferenceLinks("![id]\n\n[id]: /img.png")).toBe("![id](/img.png)\n\n"); + }); +}); + +describe("resolveReferenceLinks — owner review fixups (PR #1168)", () => { + test("does not rewrite a reference inside a fence indented 4+ spaces (block parser still treats it as code)", () => { + // The block parser detects a fence via `trimmed.startsWith('```')` after a + // full `.trim()` — ANY indentation still opens a code block. The resolver + // must recognize the exact same fence, not just fences within 3 spaces. + const md = " ```\n [a][b]\n ```\n\n[b]: https://e.com"; + // The code content must survive verbatim, and since "b" is never consumed + // outside the code fence, the definition itself must remain visible too — + // as its own trailing paragraph, not silently dropped. + expect(parseMarkdownToBlocks(md).map((b) => b.type)).toEqual(["code", "paragraph"]); + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("does not rewrite a reference inside a fence nested inside a list item at 4+ spaces", () => { + const md = "- outer\n - inner\n ```\n [a][b]\n ```\n\n[b]: https://e.com"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("protects a link definition sitting inside a
raw HTML block", () => { + const md = "
\nNotes\n\n[id]: https://from-details.com\n\n
"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("protects references and definitions inside a
 raw HTML block", () => {
+    const md = "
\n[a][b]\n
\n\n[b]: https://e.com"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("a reference used only inside a
block never counts as consumed, so the definition stays visible", () => { + const md = "
\n\n[x][id]\n\n
\n\n[id]: https://e.com"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("does not treat a GFM footnote definition ([^label]: ...) as a link reference definition", () => { + // A footnote body that is a bare token (looks exactly like a definition + // destination) is the real hazard — prose bodies with spaces already fail + // the destination shape by accident. + const md = "See the note.[^1]\n\n[^1]: https://example.com/footnote"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("does not clobber a footnote reference ([^1]) that happens to share a label with a real definition", () => { + const md = "See[^1] and [x][1]\n\n[^1]: https://footnote.com\n\n[1]: https://real-def.com"; + expect(resolveReferenceLinks(md)).toBe( + "See[^1] and [x](https://real-def.com)\n\n[^1]: https://footnote.com\n\n", + ); + }); + + test("leaves an entirely unused link reference definition visible", () => { + expect(resolveReferenceLinks("[id]: https://e.com\n\nSome unrelated text.")).toBe( + "[id]: https://e.com\n\nSome unrelated text.", + ); + }); + + test("leaves a definition visible when its only reference sits inside a fenced code block", () => { + const md = "```\n[x][id]\n```\n\n[id]: https://e.com"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("still blanks every definition line for a label once it is genuinely consumed, including redefinitions", () => { + expect( + resolveReferenceLinks("[id]: https://one.com\n[id]: https://two.com\n\n[x][id]"), + ).toBe("\n\n\n[x](https://one.com)"); + }); + + test("preserves total line count for a document mixing consumed, unused, and protected definitions", () => { + const md = [ + "# Heading", + "", + "[used]: https://used.com", + "[unused]: https://unused.com", + "", + "text [x][used]", + "", + "```", + "[y][coded]", + "```", + "", + "[coded]: https://coded.com", + ].join("\n"); + const resolved = resolveReferenceLinks(md); + expect(resolved.split("\n").length).toBe(md.split("\n").length); + const blocks = parseMarkdownToBlocks(md); + expect(blocks.find((b) => b.type === "heading")?.startLine).toBe(1); + // "unused" and "coded" (only referenced inside the fence) must remain + // visible; only the genuinely consumed "used" definition is blanked. + expect(resolved).toContain("[unused]: https://unused.com"); + expect(resolved).toContain("[coded]: https://coded.com"); + expect(resolved).not.toContain("[used]: https://used.com"); + expect(resolved).toContain("text [x](https://used.com)"); + }); + + test("supports CRLF documents and preserves the CRLF line endings", () => { + const md = "[text][id]\r\n\r\n[id]: https://e.com\r\n"; + expect(resolveReferenceLinks(md)).toBe("[text](https://e.com)\r\n\r\n\r\n"); + }); + + test("leaves an unconsumed CRLF definition visible with its line ending intact", () => { + const md = "[id]: https://e.com\r\n\r\nunrelated text\r\n"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("does not exhibit quadratic slowdown on a long run of unmatched '[' characters", () => { + const junk = "[".repeat(200_000); + const md = `${junk}\n\n[id]: https://e.com`; + const start = performance.now(); + const result = resolveReferenceLinks(md); + const elapsed = performance.now() - start; + // A naive unbounded backtracking scan would take many seconds to minutes + // here; a bounded one-pass scan finishes in well under a second. + expect(elapsed).toBeLessThan(1500); + expect(result.startsWith(junk)).toBe(true); + }); + + test("caps reference/definition label length so a single pathological label cannot force backtracking", () => { + const longLabel = "x".repeat(1500); + const md = `[text][${longLabel}]\n\n[${longLabel}]: https://e.com`; + // Deliberate safe degradation: a label above the bound is not resolved + // and its definition-shaped line is not collected either, so both sides + // are left untouched rather than partially/incorrectly rewritten. + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("is idempotent: resolving an already-resolved document is a no-op", () => { + const inputs = [ + "[text][id]\n\n[id]: https://e.com", + "![alt][id]\n\n[id]: /img.png", + "```\n[a][b]\n```\n\n[b]: https://e.com", + "- [x] done\n\n[x]: https://e.com", + "
\n[a][b]\n
\n\n[b]: https://e.com", + "[id]: https://e.com\n\nunused elsewhere", + "See[^1]\n\n[^1]: https://footnote.com", + ]; + for (const md of inputs) { + const once = resolveReferenceLinks(md); + const twice = resolveReferenceLinks(once); + expect(twice).toBe(once); + } + }); + + test("aligns tilde-fence handling with the block parser (neither treats ~~~ as a code fence)", () => { + const md = "~~~\n[a][b]\n~~~\n\n[b]: https://e.com"; + expect(parseMarkdownToBlocks(md).some((b) => b.type === "code")).toBe(false); + expect(resolveReferenceLinks(md)).toBe("~~~\n[a](https://e.com)\n~~~\n\n"); + }); + + test("resolves a destination containing a closing parenthesis", () => { + expect(resolveReferenceLinks("[x][id]\n\n[id]: https://e.com/a(b)")).toBe( + "[x](https://e.com/a(b))\n\n", + ); + }); + + test("backslash-escaped brackets never resolve, and their captured (unusable) label leaves the definition visible", () => { + const md = "Not a ref: \\[text\\]\\[id\\]\n\n[id]: https://e.com"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("a genuinely nested-bracket shortcut does not corrupt the destination pipeline or crash", () => { + // Unescaped nested brackets in link text are not legal CommonMark; the + // simplified single-pass scanner does not fully recover the intended + // reference, but it must never throw and must never do something that + // could bypass URL sanitization later. + const md = "[outer [inner] text][id]\n\n[id]: https://e.com"; + expect(() => resolveReferenceLinks(md)).not.toThrow(); + const result = resolveReferenceLinks(md); + expect(result).toContain("https://e.com"); + }); + + test("dangerous destinations still go through sanitizeLinkUrl identically to a hand-written inline link", () => { + // resolveReferenceLinks only ever emits `[text](dest)`, so it reuses the + // exact same inline-link rendering/sanitization path — it must never + // special-case or bypass it. + const resolved = resolveReferenceLinks("[x][id]\n\n[id]: javascript:alert(1)"); + expect(resolved).toBe("[x](javascript:alert(1))\n\n"); + // The literal string is unchanged (dangerous-protocol stripping happens + // downstream in sanitizeLinkUrl at render time), confirming this pass + // does not attempt — and therefore cannot get wrong — its own filtering. + }); +}); + describe("parseMarkdownToBlocks — code fences", () => { /** * Baseline: the common triple-backtick fence still works after the nested- @@ -902,6 +1220,259 @@ describe("parseMarkdownToBlocks — raw HTML blocks", () => { }); }); +describe("parseMarkdownToBlocks / resolveReferenceLinks — unclosed-HTML-opener perf (PR #1168 follow-up)", () => { + // Root cause: the balanced open/close depth scan for a multi-line HTML + // block does not advance the outer index when it fails to find a closing + // tag — so a document with many consecutive unclosed openers (e.g. + // thousands of bare `
` lines with no `
` anywhere) makes EVERY + // one of them independently re-scan all the way to end-of-document. That + // is O(N^2) work for N such lines, a real DoS-shaped hazard well within + // the 2MB annotate cap. Both markProtectedLines (used by + // resolveReferenceLinks) and parseMarkdownToBlocks's own HTML-block + // section duplicate this exact scan, so both must be fixed. + const N = 8000; + // Generous bound: a linear/bounded fix finishes in well under 100ms for + // this input; the pre-fix O(N^2) scan takes multiple seconds (measured + // ~2.2s for N=8000 during triage). 800ms leaves large machine-variance + // headroom while still failing clearly against the quadratic behavior. + const TIME_BOUND_MS = 800; + + test("parseMarkdownToBlocks stays fast with many consecutive unclosed
openers", () => { + const md = Array.from({ length: N }, () => "
").join("\n"); + const start = performance.now(); + const blocks = parseMarkdownToBlocks(md); + const elapsed = performance.now() - start; + expect(blocks).toHaveLength(N); + expect(blocks.every((b) => b.type === "html")).toBe(true); + expect(elapsed).toBeLessThan(TIME_BOUND_MS); + }); + + test("resolveReferenceLinks stays fast with many consecutive unclosed
openers", () => { + const md = + Array.from({ length: N }, () => "
").join("\n") + + "\n\n[x][id]\n\n[id]: https://e.com"; + const start = performance.now(); + const result = resolveReferenceLinks(md); + const elapsed = performance.now() - start; + expect(result).toContain("[x](https://e.com)"); + expect(elapsed).toBeLessThan(TIME_BOUND_MS); + }); + + test("parity: a real
...
block stays intact and identically protected/parsed among thousands of unclosed
decoys", () => { + const decoyCount = 5000; + const decoys = Array.from({ length: decoyCount }, () => "
").join("\n"); + const md = + `${decoys}\n\n` + + "
\nNotes\n\n[id]: https://from-details.com\n\n
" + + "\n\nAfter."; + + const start = performance.now(); + const blocks = parseMarkdownToBlocks(md); + const resolved = resolveReferenceLinks(md); + const elapsed = performance.now() - start; + expect(elapsed).toBeLessThan(TIME_BOUND_MS); + + // parseMarkdownToBlocks: the real
block is captured whole, + // undisturbed by the decoys before it, followed by its own paragraph. + const detailsBlock = blocks.find((b) => b.type === "html" && b.content.startsWith("
")); + expect(detailsBlock).toBeDefined(); + expect(detailsBlock!.content).toBe( + "
\nNotes\n\n[id]: https://from-details.com\n\n
", + ); + const paragraph = blocks.find((b) => b.type === "paragraph" && b.content === "After."); + expect(paragraph).toBeDefined(); + + // resolveReferenceLinks: the SAME
span is protected — the + // definition inside it is never consumed by anything, so it stays + // visible verbatim, exactly mirroring the block parser's own boundary + // for this block (not corrupted, not partially rewritten). + expect(resolved).toContain("[id]: https://from-details.com"); + }); +}); + +describe("parseMarkdownToBlocks / resolveReferenceLinks — long valid HTML blocks must not be truncated (owner follow-up on 9440be06)", () => { + // Regression: a fixed MAX_HTML_BLOCK_SCAN_LINES cap on the balanced + // open/close depth scan incorrectly cut off VALID HTML blocks whose + // closing tag sits beyond the cap. closeExistsFromLine already rejects a + // truly-unclosed opener in O(1) without scanning at all, so a genuinely + // closed block — however long — should simply be scanned once to its + // real end, not capped. A cap that can truncate valid parsing is not an + // acceptable trade-off no matter how generous its value. + const N_UNCLOSED = 40_000; + const TIME_BOUND_MS = 1500; + + test("a
block with its closing tag more than 2000 lines below the opener stays one whole html block", () => { + const innerLineCount = 2500; // deliberately past the old 2000-line cap + const inner = Array.from({ length: innerLineCount }, (_, i) => `body line ${i}`).join("\n"); + const md = `
\nBig\n${inner}\n
\n\nAfter.`; + + const blocks = parseMarkdownToBlocks(md); + expect(blocks).toHaveLength(2); + expect(blocks[0].type).toBe("html"); + expect(blocks[0].content).toBe(`
\nBig\n${inner}\n
`); + expect(blocks[1].type).toBe("paragraph"); + expect(blocks[1].content).toBe("After."); + }); + + test("a raw HTML block with its closing tag more than 2000 lines below the opener stays one whole html block", () => { + const rowCount = 2200; // deliberately past the old 2000-line cap + const rows = Array.from({ length: rowCount }, (_, i) => ``).join("\n"); + const md = `
${i}
\n${rows}\n
\n\nAfter table.`; + + const blocks = parseMarkdownToBlocks(md); + expect(blocks).toHaveLength(2); + expect(blocks[0].type).toBe("html"); + expect(blocks[0].content).toBe(`\n${rows}\n
`); + expect(blocks[1].type).toBe("paragraph"); + expect(blocks[1].content).toBe("After table."); + }); + + test("a link definition inside a long (>2000-line)
block stays protected by resolveReferenceLinks, and never wins over a real outside definition", () => { + const innerLineCount = 2500; + const filler = Array.from({ length: innerLineCount }, (_, i) => `body line ${i}`).join("\n"); + // Same label defined both inside the (protected) details block and + // outside it. If the interior is genuinely protected, the inside + // definition is never collected at all, so the outside one — the only + // real candidate — wins and resolves the reference. If protection were + // to end early (the truncation bug), the inside definition would be + // collected FIRST (first-definition-wins) and incorrectly win instead, + // and would incorrectly be blanked as "consumed" too. + const md = + `
\nBig\n${filler}\n\n[id]: https://inside-details.com\n\n
` + + `\n\n[id]: https://outside.com\n\ntext [x][id]`; + const resolved = resolveReferenceLinks(md); + expect(resolved).toContain(`[id]: https://inside-details.com`); // stays visible, untouched + expect(resolved).toContain("text [x](https://outside.com)"); // outside definition wins + expect(resolved).not.toContain("[id]: https://outside.com\n"); // the real (consumed) one is blanked + }); + + test("40k unclosed
openers (no close anywhere) stay fast, with parity between parser and resolver", () => { + const decoys = Array.from({ length: N_UNCLOSED }, () => "
").join("\n"); + const md = `${decoys}\n\n[x][id]\n\n[id]: https://e.com`; + + const parseStart = performance.now(); + const blocks = parseMarkdownToBlocks(md); + const parseElapsed = performance.now() - parseStart; + + const resolveStart = performance.now(); + const resolved = resolveReferenceLinks(md); + const resolveElapsed = performance.now() - resolveStart; + + expect(parseElapsed).toBeLessThan(TIME_BOUND_MS); + expect(resolveElapsed).toBeLessThan(TIME_BOUND_MS); + + // Parity: every decoy is its own single-line html block to the parser... + expect(blocks.filter((b) => b.type === "html")).toHaveLength(N_UNCLOSED); + // ...and every decoy line is likewise individually protected (never + // rewritten) by the resolver — same boundary, both call sites agree. + expect(resolved).toContain("[x](https://e.com)"); + expect(resolved.split("\n").filter((l) => l === "
")).toHaveLength(N_UNCLOSED); + }); + + test("a long valid
block survives even when preceded by thousands of unclosed
decoys", () => { + const decoyCount = 5000; + const decoys = Array.from({ length: decoyCount }, () => "
").join("\n"); + const innerLineCount = 2500; + const inner = Array.from({ length: innerLineCount }, (_, i) => `body line ${i}`).join("\n"); + const md = `${decoys}\n\n
\nBig\n${inner}\n
\n\nAfter.`; + + const start = performance.now(); + const blocks = parseMarkdownToBlocks(md); + const elapsed = performance.now() - start; + expect(elapsed).toBeLessThan(TIME_BOUND_MS); + + const detailsBlock = blocks.find((b) => b.type === "html" && b.content.startsWith("
")); + expect(detailsBlock).toBeDefined(); + expect(detailsBlock!.content).toBe(`
\nBig\n${inner}\n
`); + const paragraph = blocks.find((b) => b.type === "paragraph" && b.content === "After."); + expect(paragraph).toBeDefined(); + }); +}); + +describe("parseMarkdownToBlocks / resolveReferenceLinks — linear matching-close index (owner follow-up on a55db2b9)", () => { + // Owner-flagged regression: removing the fixed cap fixed truncation but + // reintroduced O(N^2) for a different adversarial shape — N unclosed + // `
` openers followed by a SINGLE trailing `
`. + // closeExistsFromLine's "does a close exist anywhere" pre-check is true + // for every one of the N openers (the trailing close exists), so every + // one of them still independently scans forward — most all the way to + // end-of-document — before giving up. Measured (pre-fix): 5000 → ~1.0s, + // 10000 → ~4.1s (textbook ~4x per doubling). Fixed by replacing the + // scan entirely with a per-tag-name prefix-sum + "next smaller-or-equal + // element" index (a classic O(N) monotonic-stack construction, built once + // per tag name and cached per document), so every opener's closing + // position — whether it exists, and exactly where if so, however far away + // — is an O(1) lookup with no scanning at all. + const N = 40_000; + const TIME_BOUND_MS = 1500; + + test("N unclosed
openers followed by one trailing
stay fast", () => { + const md = Array.from({ length: N }, () => "
").join("\n") + "\n
"; + + const parseStart = performance.now(); + const blocks = parseMarkdownToBlocks(md); + const parseElapsed = performance.now() - parseStart; + expect(parseElapsed).toBeLessThan(TIME_BOUND_MS); + + // Only the LAST opener (immediately preceding the trailing close) can + // actually pair with it — every earlier opener's cumulative depth + // overshoots and never returns to its own baseline, so it stays an + // unclosed, single-line block. N-1 singles + 1 paired block = N blocks. + expect(blocks).toHaveLength(N); + expect(blocks.slice(0, N - 1).every((b) => b.type === "html" && b.content === "
")).toBe( + true, + ); + expect(blocks[N - 1].type).toBe("html"); + expect(blocks[N - 1].content).toBe("
\n
"); + }); + + test("resolveReferenceLinks stays fast and agrees with the parser on the same N-openers-plus-one-close document", () => { + const md = + Array.from({ length: N }, () => "
").join("\n") + + "\n
\n\n[x][id]\n\n[id]: https://e.com"; + + const start = performance.now(); + const resolved = resolveReferenceLinks(md); + const elapsed = performance.now() - start; + expect(elapsed).toBeLessThan(TIME_BOUND_MS); + expect(resolved).toContain("[x](https://e.com)"); + // Every decoy line and the paired
/
stay literal (protected), + // exactly mirroring the parser's block boundaries above. + expect(resolved.split("\n").filter((l) => l === "
")).toHaveLength(N); + expect(resolved).toContain("
"); + }); + + test("nested same-tag blocks still balance correctly (depth, not just presence, matters)", () => { + const md = + "
\nOuter\n
\nInner\n
\nouter tail\n
"; + const blocks = parseMarkdownToBlocks(md); + expect(blocks).toHaveLength(1); + expect(blocks[0].type).toBe("html"); + expect(blocks[0].content).toBe(md); + }); + + test("mixed tag types nest independently — a inside a
does not confuse the details/details matcher", () => { + const md = + "
\nNotes\n
\n\n
1
\nafter table\n
\n\nAfter."; + const blocks = parseMarkdownToBlocks(md); + expect(blocks).toHaveLength(2); + expect(blocks[0].type).toBe("html"); + expect(blocks[0].content).toBe( + "
\nNotes\n\n\n
1
\nafter table\n
", + ); + expect(blocks[1].content).toBe("After."); + }); + + test("a valid >2000-line
block still survives (no truncating cap reintroduced)", () => { + const innerLineCount = 3000; + const inner = Array.from({ length: innerLineCount }, (_, i) => `body line ${i}`).join("\n"); + const md = `
\nBig\n${inner}\n
\n\nAfter.`; + const blocks = parseMarkdownToBlocks(md); + expect(blocks).toHaveLength(2); + expect(blocks[0].content).toBe(`
\nBig\n${inner}\n
`); + }); +}); + describe("computeListIndices", () => { test("all unordered → all null", () => { const blocks = [li(0, false), li(0, false), li(0, false)]; diff --git a/packages/ui/utils/parser.ts b/packages/ui/utils/parser.ts index 540379c99..1e918c806 100644 --- a/packages/ui/utils/parser.ts +++ b/packages/ui/utils/parser.ts @@ -204,19 +204,362 @@ export interface ParseMarkdownOptions { frontmatter?: boolean; } +// CommonMark bounds a link label to 999 characters. Reusing that bound here +// also caps the worst-case backtracking cost of the bracket-matching groups +// below to a constant per starting position, turning a document with a very +// long run of unmatched `[` characters (a real hazard within the 2MB annotate +// cap) into a linear scan instead of a quadratic one. A label longer than +// this is a deliberate, documented degradation: it is neither collected as a +// definition nor resolved as a reference, so it is simply left untouched +// rather than partially or incorrectly rewritten. +const MAX_REF_LABEL_CHARS = 999; +// Same reasoning applied to the inline-code-span alternative: bounding how far +// a lazy scan for a closing backtick run can travel keeps a line with many +// stray, unterminated backticks linear too. 5000 is far beyond any realistic +// inline code span, so legitimate spans are unaffected. +const MAX_CODE_SPAN_CHARS = 5000; +// Defense-in-depth cap on the number of definitions collected from a single +// document. A pathological document could otherwise grow the map without +// bound; this keeps that growth bounded even though ordinary documents never +// approach it. +const MAX_TRACKED_DEFINITIONS = 20_000; + +// A link reference definition: `[label]: destination "optional title"`, with up +// to three leading spaces. The destination is a bare token or an <...> form; any +// trailing text must be a quoted or parenthesized title, otherwise the line is +// ordinary prose (so `[Reminder]: call the bank` is NOT a definition). Matches +// the CommonMark shape closely enough for the simplified parser. `\r?` before +// the end anchor tolerates a CRLF source (lines are split on `\n` only, so a +// CRLF line keeps its trailing `\r`). +const REFERENCE_DEFINITION_RE = new RegExp( + `^ {0,3}\\[([^\\]]{1,${MAX_REF_LABEL_CHARS}})\\]:[ \\t]*(?:<([^>]*)>|(\\S+))[ \\t]*(?:"[^"]*"|'[^']*'|\\([^)]*\\))?[ \\t]*\\r?$`, +); + +// One left-to-right pass over a line. The first alternative matches a whole +// inline code span (balanced backtick run) so its contents are skipped; the +// second matches a reference link/image: optional `!`, the bracketed text, then +// an optional second bracket for the full (`[label]`) or collapsed (`[]`) forms. +// A bare `[text]` is the shortcut form, resolved only when it names a definition +// and is not actually an inline link. Groups: 1 code ticks, 2 `!`, 3 text, +// 4 second bracket, 5 label. +const REFERENCE_LINK_RE = new RegExp( + `(\`+)[^\\n]{0,${MAX_CODE_SPAN_CHARS}}?\\1|(!?)\\[([^\\]]{1,${MAX_REF_LABEL_CHARS}})\\](\\[([^\\]]{0,${MAX_REF_LABEL_CHARS}})\\])?`, + 'g', +); + +// CommonMark label matching is case-insensitive and collapses internal runs of +// whitespace. +const normalizeRefLabel = (label: string): string => + label.trim().replace(/\s+/g, ' ').toLowerCase(); + +/** + * One pass over the lines that marks every line the block parser (below) will + * render as code or raw HTML — fenced code blocks and HTML blocks — so link + * reference definitions and references inside them are left completely + * untouched. This reuses the exact same conditions the block parser itself + * uses (not a looser approximation), so the two can never disagree about + * where code/HTML starts and ends: + * + * - Fences: `trimmed.startsWith('```')` after a full `.trim()` — the block + * parser has no minimum-indent exemption, so ANY indentation (a fence + * nested inside a list item, or simply indented 4+ spaces) still opens a + * code block, and this must too. Only backtick fences are recognized — + * the block parser has no `~~~` support, so this doesn't either (a `~~~` + * line is ordinary text to both). + * - Raw HTML blocks: the same `HTML_BLOCK_OPEN_RE`/`HTML_BLOCK_TAGS`/ + * `VOID_HTML_TAGS` the block parser uses, with the same three extents + * (blank-line termination for a leading close tag, single-line for void + * tags, balanced-depth scanning otherwise) — so a definition sitting + * inside `
` or `
` is protected exactly as + * far as the block parser's own HTML block extends. + */ +/** + * Per-tag-name index backing `findHtmlBlockEnd`. `augmented` is the running + * open-tag-count-minus-close-tag-count prefix sum for this tag name, with a + * virtual baseline of 0 prepended at index 0 — so `augmented[k]` is the sum + * through line `k-1` (the depth baseline a block opening at line `k` must + * return to) and `augmented[k+1]` is the sum through line `k`. + * `nextAtOrBelow[m]` is the classic "next element at or below this one" + * index over `augmented`: the smallest `m' > m` with `augmented[m'] <= + * augmented[m]`, or -1 if none exists. + */ +interface TagCloseIndex { + augmented: number[]; + nextAtOrBelow: number[]; +} + +/** + * Builds a `TagCloseIndex` for one tag name in a single O(N) pass (plus a + * classic O(N) monotonic-stack pass for `nextAtOrBelow` — each index is + * pushed and popped at most once, so the two passes together are linear in + * the document's line count, independent of how many opening/closing tags + * it contains). + */ +function buildTagCloseIndex(lines: string[], tagName: string): TagCloseIndex { + const openRe = new RegExp(`<${tagName}(?:\\s|>|/|$)`, 'gi'); + const closeRe = new RegExp(``, 'gi'); + const n = lines.length; + const augmented = new Array(n + 1); + augmented[0] = 0; + let running = 0; + for (let k = 0; k < n; k++) { + running += (lines[k].match(openRe) || []).length; + running -= (lines[k].match(closeRe) || []).length; + augmented[k + 1] = running; + } + const nextAtOrBelow = new Array(n + 1).fill(-1); + const stack: number[] = []; + for (let m = n; m >= 0; m--) { + while (stack.length && augmented[stack[stack.length - 1]] > augmented[m]) stack.pop(); + nextAtOrBelow[m] = stack.length ? stack[stack.length - 1] : -1; + stack.push(m); + } + return { augmented, nextAtOrBelow }; +} + +/** + * Shared helper computing the last line index of a balanced open/close-tag + * HTML block that opens at `startIndex` with the given already-computed + * `depth` (the opening line's own open-tag count minus close-tag count). + * Used by both `markProtectedLines` (the resolver's protection pass) and + * `parseMarkdownToBlocks` (the block parser) so the two can never disagree + * about a multi-line HTML block's extent, and so a fix here lives in exactly + * one place instead of two copies drifting apart. + * + * History: naively scanning line-by-line from `startIndex` until depth + * returns to zero (or giving up at end-of-document) is O(N^2) for a + * document with many consecutive unclosed openers (e.g. thousands of bare + * `
` lines), since every one of them re-scans to EOF. A first fix + * added an O(1) "does a close exist anywhere" pre-check plus a fixed + * line-count cap on the residual scan — but that cap silently truncated + * VALID blocks longer than it, and removing the cap alone reopened a + * closely related O(N^2) case: N unclosed openers followed by a SINGLE + * trailing close still all pass the "a close exists somewhere" pre-check, + * so every one of them still scans forward (mostly to EOF) before giving up. + * + * Fixed properly here with a per-tag-name prefix-sum index + * (`buildTagCloseIndex`, O(N), built once per tag name and cached per + * document — see `closeCache`): finding "the exact line where a block + * starting at `startIndex` closes, if ever" is exactly the classic "next + * smaller-or-equal element" query against that prefix sum, which the index + * answers in O(1). No scanning happens per opener at all — not for a block + * that never closes, not for one that closes after any number of + * intervening lines, however many. This is provably linear overall (a + * document with T distinct protected tag names costs O(T * N) to index, + * and T is bounded by the small, fixed `HTML_BLOCK_TAGS` set) and can never + * truncate a valid block, because it always finds the block's real end + * (however far away) rather than giving up at a fixed distance. + * + * Returns `startIndex` unchanged when the block never closes: depth <= 0, + * or the running depth never returns to exactly zero anywhere in the rest + * of the document (whether because no close exists at all, or one exists + * but is insufficient to bring the count back to exactly the opener's own + * baseline — e.g. an unbalanced/self-closing tag). + */ +function findHtmlBlockEnd( + lines: string[], + startIndex: number, + tagName: string, + depth: number, + closeCache: Map, +): number { + if (depth <= 0) return startIndex; + let index = closeCache.get(tagName); + if (!index) { + index = buildTagCloseIndex(lines, tagName); + closeCache.set(tagName, index); + } + const { augmented, nextAtOrBelow } = index; + const m = nextAtOrBelow[startIndex]; + if (m === -1) return startIndex; + return augmented[m] === augmented[startIndex] ? m - 1 : startIndex; +} + +const markProtectedLines = (lines: string[]): boolean[] => { + const isProtected = new Array(lines.length).fill(false); + let fenceLen = 0; // 0 = not currently inside a fence + const closeCache = new Map(); + for (let i = 0; i < lines.length; i++) { + if (fenceLen > 0) { + isProtected[i] = true; + if (new RegExp('^\\s*`{' + fenceLen + ',}').test(lines[i])) fenceLen = 0; + continue; + } + const trimmed = lines[i].trim(); + if (trimmed.startsWith('```')) { + fenceLen = trimmed.match(/^`+/)![0].length; + isProtected[i] = true; + continue; + } + const htmlTagMatch = trimmed.match(HTML_BLOCK_OPEN_RE); + if (htmlTagMatch && HTML_BLOCK_TAGS.has(htmlTagMatch[1].toLowerCase())) { + const tagName = htmlTagMatch[1].toLowerCase(); + const isCloseTag = trimmed.startsWith('') && i + 1 < lines.length && lines[i + 1].trim() !== '') { + i++; + isProtected[i] = true; + } + } else { + const openRe = new RegExp(`<${tagName}(?:\\s|>|/|$)`, 'gi'); + const closeRe = new RegExp(``, 'gi'); + const depth = (lines[i].match(openRe) || []).length - (lines[i].match(closeRe) || []).length; + const end = findHtmlBlockEnd(lines, i, tagName, depth, closeCache); + if (end > i) { + for (let idx = i + 1; idx <= end; idx++) isProtected[idx] = true; + i = end; + } + } + } + } + return isProtected; +}; + +/** Resolve reference links/images in one non-code, non-HTML line. A single + * left-to-right pass: an inline code span is matched as a whole and returned + * verbatim, so a reference-looking pattern inside backticks is never + * rewritten; only bracketed references outside code are resolved. Every label + * that actually resolves against a definition is recorded into `usedLabels`, + * so the caller can tell a genuinely consumed definition from an unused one. */ +const resolveRefsInLine = ( + line: string, + defs: Map, + usedLabels: Set, +): string => { + if (!line.includes('[')) return line; + return line.replace( + REFERENCE_LINK_RE, + (match, codeTicks, bang, text, secondBracket, label, offset: number, whole: string) => { + if (codeTicks !== undefined) return match; // inline code span: keep verbatim + let refLabel: string; + if (secondBracket === undefined) { + // Shortcut `[text]`: not a link when an inline `(...)` destination + // follows (that is an inline link the existing renderer already draws). + if (whole[offset + match.length] === '(') return match; + // Nor when it is a task-list checkbox marker at the start of a list + // item (`- [x]`); the checkbox parser owns that `[x]`, and resolving it + // against a stray `x`/`X` definition would clobber the item. + if (/^[ xX]$/.test(text) && /^\s*(?:[-*+]|\d+[.)])\s+$/.test(whole.slice(0, offset))) { + return match; + } + refLabel = text; + } else { + refLabel = label === '' ? text : label; + } + const normalized = normalizeRefLabel(refLabel); + const dest = defs.get(normalized); + // An unknown reference stays literal, matching CommonMark and avoiding + // false links for bracketed prose like `[TODO]` or array indices. + if (!dest) return match; + usedLabels.add(normalized); + return `${bang}[${text}](${dest})`; + }, + ); +}; + +/** + * Resolve CommonMark link reference definitions and reference links into inline + * `[text](url)` links, so the shared inline renderer draws them instead of + * showing raw `[text][id]` and `[id]: url` text (issue #923). Definitions and + * references inside fenced code blocks, raw HTML blocks, and inline code spans + * are left untouched. A definition-shaped line is only ever blanked when its + * label was actually consumed by a resolved reference outside a protected + * region — an unused definition, or one referenced only from inside code/HTML, + * stays visible exactly as written. Blanked lines keep block start-line + * numbers accurate (and their own CRLF ending, so line endings round-trip). + * GFM footnote definitions (`[^label]: ...`) are never treated as link + * definitions. No-op (returns the input) when the document defines no + * (non-footnote) references. + */ +export const resolveReferenceLinks = (markdown: string): string => { + if (!markdown.includes('[')) return markdown; + const lines = markdown.split('\n'); + const isProtected = markProtectedLines(lines); + const defs = new Map(); + // The normalized label a definition-shaped line defines, or null if the + // line isn't a definition (or is a footnote definition, which is never + // collected/blanked). + const defLabelByLine = new Array(lines.length).fill(null); + // A definition cannot interrupt a paragraph (CommonMark 4.7): a line matching + // the definition shape is only a definition when it can start a block, i.e. + // the previous line is the document start, blank, a protected code/HTML + // line (each is its own block), or itself a definition. Otherwise the line + // is paragraph continuation text and must be left untouched, or a bare + // `[word]: token` under a sentence would be silently deleted. + let canStartDefinition = true; + for (let i = 0; i < lines.length; i++) { + if (isProtected[i]) { + canStartDefinition = true; + continue; + } + const blank = lines[i].trim() === ''; + const match = canStartDefinition && !blank ? lines[i].match(REFERENCE_DEFINITION_RE) : null; + if (match) { + const rawLabel = match[1]; + // GFM footnote definition ([^label]: ...) — not a link reference + // definition. Leave it out of `defs` entirely so it can never be + // collected, blanked, or accidentally satisfy a footnote reference's + // lookup; it stays block-starting like any other definition line. + if (!rawLabel.startsWith('^') && defs.size < MAX_TRACKED_DEFINITIONS) { + const label = normalizeRefLabel(rawLabel); + const dest = match[2] !== undefined ? match[2] : match[3]; + // First definition wins, per CommonMark. + if (label && dest && !defs.has(label)) defs.set(label, dest); + defLabelByLine[i] = label; + } + // A run of definitions stays eligible; canStartDefinition remains true. + } else { + // Blank keeps a new block startable; any other non-definition line starts + // (or continues) a paragraph, so a following definition-shaped line is text. + canStartDefinition = blank; + } + } + if (defs.size === 0) return markdown; + const usedLabels = new Set(); + // Resolve references first; definition-shaped lines are passed through + // unresolved (never fed to resolveRefsInLine) so a definition's own + // `[label]` can never be mistaken for a reference to itself. + const resolved = lines.map((line, i) => + isProtected[i] || defLabelByLine[i] !== null ? line : resolveRefsInLine(line, defs, usedLabels), + ); + return resolved + .map((line, i) => { + const label = defLabelByLine[i]; + if (label === null || !usedLabels.has(label)) return line; + // Blank in place, preserving this line's own CRLF ending if it had one. + return line.endsWith('\r') ? '\r' : ''; + }) + .join('\n'); +}; + /** * A simplified markdown parser that splits content into linear blocks. * For a production app, we would use a robust AST walker (remark), * but for this demo, we want predictable text-anchoring. */ export const parseMarkdownToBlocks = (markdown: string, options?: ParseMarkdownOptions): Block[] => { - const { content: cleanMarkdown, contentStartLine } = + const { content: rawContent, contentStartLine } = options?.frontmatter === false ? { content: markdown, contentStartLine: 1 } : extractFrontmatter(markdown); + // Resolve link reference definitions into inline links before splitting. This + // blanks definition lines in place, so line count (and every block's + // startLine) is preserved. + const cleanMarkdown = resolveReferenceLinks(rawContent); const lines = cleanMarkdown.split('\n'); const blocks: Block[] = []; let currentId = 0; + // Cache for findHtmlBlockEnd's per-tag-name prefix-sum index — scoped per + // parse call (per document) and shared across every HTML-block opener + // encountered below, so a document with many consecutive openers of the + // same tag only pays its one-time O(N) build cost once. + const htmlCloseCache = new Map(); let buffer: string[] = []; let currentType: Block['type'] = 'paragraph'; @@ -607,23 +950,15 @@ export const parseMarkdownToBlocks = (markdown: string, options?: ParseMarkdownO const openRe = new RegExp(`<${tagName}(?:\\s|>|/|$)`, 'gi'); const closeRe = new RegExp(``, 'gi'); const depth = (line.match(openRe) || []).length - (line.match(closeRe) || []).length; - if (depth > 0) { - // Scan ahead for the matching close tag. If none is ever found — a - // self-closing