From 6211371cf261356f1b0c2ef88162190d335844bf Mon Sep 17 00:00:00 2001 From: rNoz Date: Fri, 31 Jul 2026 17:56:26 +0200 Subject: [PATCH 1/5] feat(ui): render markdown reference links The simplified markdown parser only understood inline links `[text](url)`, so CommonMark reference links rendered as raw text: `[text][id]` and the `[id]: url` definition both showed literally (#923). Add `resolveReferenceLinks`, a pure pass run at the top of `parseMarkdownToBlocks` that rewrites full (`[text][id]`), collapsed (`[text][]`), and shortcut (`[text]`) references, plus their image forms, into inline `[text](url)` links, so the existing inline renderer draws them. Link reference definitions are collected first (first definition wins, labels matched case-insensitively with collapsed whitespace, `` and quoted-title forms supported) and then blanked in place, so a definition never renders and every block keeps its original source line number. Resolution is code-aware: references and definitions inside fenced code blocks and inline code spans are left verbatim, a shortcut is skipped when an inline `(...)` destination follows it or when it is a task-list checkbox marker at the start of a list item, and an unknown reference stays literal so bracketed prose like `[TODO]` or `[0]` never becomes a false link. A definition-shaped line is only collected when it can start a block (after a blank line, a code fence, another definition, or the document start), so a `[word]: token` line that continues a paragraph is left as text rather than deleted (CommonMark: a definition cannot interrupt a paragraph). A document with no definitions is returned unchanged. --- packages/ui/utils/parser.test.ts | 127 +++++++++++++++++++++++++++- packages/ui/utils/parser.ts | 140 ++++++++++++++++++++++++++++++- 2 files changed, 265 insertions(+), 2 deletions(-) diff --git a/packages/ui/utils/parser.test.ts b/packages/ui/utils/parser.test.ts index f7f148039..b1e6341b3 100644 --- a/packages/ui/utils/parser.test.ts +++ b/packages/ui/utils/parser.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect } from "bun:test"; -import { parseMarkdownToBlocks, computeListIndices, extractFrontmatter, exportAnnotations } from "./parser"; +import { parseMarkdownToBlocks, computeListIndices, extractFrontmatter, exportAnnotations, resolveReferenceLinks } from "./parser"; import { shouldStripFrontmatter } from "@plannotator/core/annotatable"; import type { Block } from "../types"; @@ -15,6 +15,131 @@ const li = (level: number, ordered: boolean, orderedStart?: number): Block => ({ startLine: 1, }); +describe("resolveReferenceLinks (#923)", () => { + test("resolves full, collapsed, and shortcut references to inline links", () => { + expect(resolveReferenceLinks("[text][id]\n\n[id]: https://e.com")).toBe( + "[text](https://e.com)\n\n", + ); + expect(resolveReferenceLinks("[text][]\n\n[text]: https://e.com")).toBe( + "[text](https://e.com)\n\n", + ); + expect(resolveReferenceLinks("[text]\n\n[text]: https://e.com")).toBe( + "[text](https://e.com)\n\n", + ); + }); + + test("matches labels case-insensitively and collapses internal whitespace", () => { + expect(resolveReferenceLinks("[Text][My Ref]\n\n[my ref]: https://e.com")).toBe( + "[Text](https://e.com)\n\n", + ); + }); + + test("supports the angle-bracket destination form and reference images", () => { + expect(resolveReferenceLinks("[x][id]\n\n[id]: ")).toBe( + "[x](https://e.com)\n\n", + ); + expect(resolveReferenceLinks("![alt][id]\n\n[id]: /img.png")).toBe( + "![alt](/img.png)\n\n", + ); + }); + + test("uses the first definition when a label is defined more than once", () => { + expect( + resolveReferenceLinks("[id]: https://one.com\n[id]: https://two.com\n\n[x][id]"), + ).toBe("\n\n\n[x](https://one.com)"); + }); + + test("leaves unknown references and non-reference brackets untouched", () => { + expect(resolveReferenceLinks("[text][missing].")).toBe("[text][missing]."); + // No definitions at all: fast path returns the input unchanged. + expect(resolveReferenceLinks("array [0] and [TODO] here")).toBe( + "array [0] and [TODO] here", + ); + // A shortcut that does not name a definition stays literal even when other + // definitions exist. + expect(resolveReferenceLinks("[TODO] and [0]\n\n[id]: https://e.com")).toBe( + "[TODO] and [0]\n\n", + ); + }); + + test("does not double-link an inline link whose text matches a definition", () => { + expect( + resolveReferenceLinks("[text](https://real.com)\n\n[text]: https://def.com"), + ).toBe("[text](https://real.com)\n\n"); + }); + + test("never rewrites references inside fenced code blocks or inline code spans", () => { + expect(resolveReferenceLinks("```\n[a][b]\n```\n\n[b]: https://e.com")).toBe( + "```\n[a][b]\n```\n\n", + ); + expect(resolveReferenceLinks("use `[a][b]` here\n\n[b]: https://e.com")).toBe( + "use `[a][b]` here\n\n", + ); + }); + + test("does not collect a definition that sits inside a fenced code block", () => { + // The only `[id]:` is inside code, so `[id]` outside stays a literal shortcut. + expect( + resolveReferenceLinks("```\n[id]: https://code.com\n```\n\n[id]"), + ).toBe("```\n[id]: https://code.com\n```\n\n[id]"); + }); + + test("does not treat prose with an invalid title as a definition", () => { + expect( + resolveReferenceLinks("[Reminder]: call the bank tomorrow\n\n[Reminder]"), + ).toBe("[Reminder]: call the bank tomorrow\n\n[Reminder]"); + }); + + test("does not corrupt bare space-delimited numbers on a resolved line", () => { + expect(resolveReferenceLinks("value is 0 and 1 and [x][id]\n\n[id]: https://e.com")).toBe( + "value is 0 and 1 and [x](https://e.com)\n\n", + ); + }); + + test("blanks definition lines in place so block start-lines stay accurate", () => { + const blocks = parseMarkdownToBlocks("[id]: https://e.com\n\n# Heading\n\ntext [x][id]"); + // The definition line renders nothing; the heading and paragraph keep their + // original source line numbers. + const heading = blocks.find((b) => b.type === "heading"); + const paragraph = blocks.find((b) => b.type === "paragraph"); + expect(heading?.startLine).toBe(3); + expect(paragraph?.startLine).toBe(5); + expect(paragraph?.content).toBe("text [x](https://e.com)"); + expect(blocks.some((b) => b.content.includes("[id]: https://e.com"))).toBe(false); + }); + + test("does not delete a definition-shaped line that continues a paragraph", () => { + // CommonMark: a definition cannot interrupt a paragraph. The second line + // must survive as content, not be silently blanked. + expect(resolveReferenceLinks("The config keys are:\n[timeout]: 30")).toBe( + "The config keys are:\n[timeout]: 30", + ); + expect(resolveReferenceLinks("text before\n[id]: url\nmore [x][id]")).toBe( + "text before\n[id]: url\nmore [x][id]", + ); + }); + + test("collects a definition after a blank line, a code fence, or another definition", () => { + expect(resolveReferenceLinks("[a]: https://one.com\n[b]: https://two.com\n\n[x][a] [y][b]")).toBe( + "\n\n\n[x](https://one.com) [y](https://two.com)", + ); + expect(resolveReferenceLinks("```\ncode\n```\n[id]: https://e.com\n\n[x][id]")).toBe( + "```\ncode\n```\n\n\n[x](https://e.com)", + ); + }); + + test("does not clobber a checked task-list item when an [x] definition exists", () => { + expect(resolveReferenceLinks("- [x] done task\n- [ ] todo\n\n[x]: https://e.com")).toBe( + "- [x] done task\n- [ ] todo\n\n", + ); + expect(resolveReferenceLinks("1. [x] done\n\n[x]: https://e.com")).toBe("1. [x] done\n\n"); + }); + + test("resolves the shortcut image form", () => { + expect(resolveReferenceLinks("![id]\n\n[id]: /img.png")).toBe("![id](/img.png)\n\n"); + }); +}); + describe("parseMarkdownToBlocks — code fences", () => { /** * Baseline: the common triple-backtick fence still works after the nested- diff --git a/packages/ui/utils/parser.ts b/packages/ui/utils/parser.ts index 540379c99..c221dc179 100644 --- a/packages/ui/utils/parser.ts +++ b/packages/ui/utils/parser.ts @@ -204,16 +204,154 @@ export interface ParseMarkdownOptions { frontmatter?: boolean; } +// A link reference definition: `[label]: destination "optional title"`, with up +// to three leading spaces. The destination is a bare token or an <...> form; any +// trailing text must be a quoted or parenthesized title, otherwise the line is +// ordinary prose (so `[Reminder]: call the bank` is NOT a definition). Matches +// the CommonMark shape closely enough for the simplified parser. +const REFERENCE_DEFINITION_RE = + /^ {0,3}\[([^\]]+)\]:[ \t]*(?:<([^>]*)>|(\S+))[ \t]*(?:"[^"]*"|'[^']*'|\([^)]*\))?[ \t]*$/; + +// One left-to-right pass over a line. The first alternative matches a whole +// inline code span (balanced backtick run) so its contents are skipped; the +// second matches a reference link/image: optional `!`, the bracketed text, then +// an optional second bracket for the full (`[label]`) or collapsed (`[]`) forms. +// A bare `[text]` is the shortcut form, resolved only when it names a definition +// and is not actually an inline link. Groups: 1 code ticks, 2 `!`, 3 text, +// 4 second bracket, 5 label. +const REFERENCE_LINK_RE = /(`+).+?\1|(!?)\[([^\]]+)\](\[([^\]]*)\])?/g; + +// CommonMark label matching is case-insensitive and collapses internal runs of +// whitespace. +const normalizeRefLabel = (label: string): string => + label.trim().replace(/\s+/g, ' ').toLowerCase(); + +/** + * Mark every line that sits inside a fenced code block (opener, content, and + * closer), so link reference definitions and links inside code are left + * untouched. Mirrors the variable-length ``` / ~~~ fence handling in the block + * loop below. + */ +const markFencedLines = (lines: string[]): boolean[] => { + const fenced = new Array(lines.length).fill(false); + let fenceChar = ''; + let fenceLen = 0; + for (let i = 0; i < lines.length; i++) { + const trimmed = lines[i].replace(/^ {0,3}/, ''); + const opener = trimmed.match(/^(`{3,}|~{3,})/); + if (fenceLen === 0) { + if (opener) { + fenceChar = opener[1][0]; + fenceLen = opener[1].length; + fenced[i] = true; + } + } else { + fenced[i] = true; + const closer = new RegExp('^' + fenceChar + '{' + fenceLen + ',}[ \\t]*$'); + if (opener && opener[1][0] === fenceChar && closer.test(trimmed)) { + fenceChar = ''; + fenceLen = 0; + } + } + } + return fenced; +}; + +/** Resolve reference links/images in one non-code line. A single left-to-right + * pass: an inline code span is matched as a whole and returned verbatim, so a + * reference-looking pattern inside backticks is never rewritten; only bracketed + * references outside code are resolved. */ +const resolveRefsInLine = (line: string, defs: Map): string => { + if (!line.includes('[')) return line; + return line.replace( + REFERENCE_LINK_RE, + (match, codeTicks, bang, text, secondBracket, label, offset: number, whole: string) => { + if (codeTicks !== undefined) return match; // inline code span: keep verbatim + let refLabel: string; + if (secondBracket === undefined) { + // Shortcut `[text]`: not a link when an inline `(...)` destination + // follows (that is an inline link the existing renderer already draws). + if (whole[offset + match.length] === '(') return match; + // Nor when it is a task-list checkbox marker at the start of a list + // item (`- [x]`); the checkbox parser owns that `[x]`, and resolving it + // against a stray `x`/`X` definition would clobber the item. + if (/^[ xX]$/.test(text) && /^\s*(?:[-*+]|\d+[.)])\s+$/.test(whole.slice(0, offset))) { + return match; + } + refLabel = text; + } else { + refLabel = label === '' ? text : label; + } + const dest = defs.get(normalizeRefLabel(refLabel)); + // An unknown reference stays literal, matching CommonMark and avoiding + // false links for bracketed prose like `[TODO]` or array indices. + return dest ? `${bang}[${text}](${dest})` : match; + }, + ); +}; + +/** + * Resolve CommonMark link reference definitions and reference links into inline + * `[text](url)` links, so the shared inline renderer draws them instead of + * showing raw `[text][id]` and `[id]: url` text (issue #923). Definitions and + * references inside fenced code blocks and inline code spans are left untouched. + * Definition lines are blanked in place rather than removed so block start-line + * numbers stay accurate. No-op (returns the input) when the document defines no + * references. + */ +export const resolveReferenceLinks = (markdown: string): string => { + if (!markdown.includes('[')) return markdown; + const lines = markdown.split('\n'); + const fenced = markFencedLines(lines); + const defs = new Map(); + const isDefLine = new Array(lines.length).fill(false); + // A definition cannot interrupt a paragraph (CommonMark 4.7): a line matching + // the definition shape is only a definition when it can start a block, i.e. + // the previous line is the document start, blank, a fenced-code line (a code + // block is its own block), or itself a definition. Otherwise the line is + // paragraph continuation text and must be left untouched, or a bare + // `[word]: token` under a sentence would be silently deleted. + let canStartDefinition = true; + for (let i = 0; i < lines.length; i++) { + if (fenced[i]) { + canStartDefinition = true; + continue; + } + const blank = lines[i].trim() === ''; + const match = canStartDefinition && !blank ? lines[i].match(REFERENCE_DEFINITION_RE) : null; + if (match) { + const label = normalizeRefLabel(match[1]); + const dest = match[2] !== undefined ? match[2] : match[3]; + // First definition wins, per CommonMark. + if (label && dest && !defs.has(label)) defs.set(label, dest); + isDefLine[i] = true; + // A run of definitions stays eligible; canStartDefinition remains true. + } else { + // Blank keeps a new block startable; any other non-definition line starts + // (or continues) a paragraph, so a following definition-shaped line is text. + canStartDefinition = blank; + } + } + if (defs.size === 0) return markdown; + return lines + .map((line, i) => (isDefLine[i] ? '' : fenced[i] ? line : resolveRefsInLine(line, defs))) + .join('\n'); +}; + /** * A simplified markdown parser that splits content into linear blocks. * For a production app, we would use a robust AST walker (remark), * but for this demo, we want predictable text-anchoring. */ export const parseMarkdownToBlocks = (markdown: string, options?: ParseMarkdownOptions): Block[] => { - const { content: cleanMarkdown, contentStartLine } = + const { content: rawContent, contentStartLine } = options?.frontmatter === false ? { content: markdown, contentStartLine: 1 } : extractFrontmatter(markdown); + // Resolve link reference definitions into inline links before splitting. This + // blanks definition lines in place, so line count (and every block's + // startLine) is preserved. + const cleanMarkdown = resolveReferenceLinks(rawContent); const lines = cleanMarkdown.split('\n'); const blocks: Block[] = []; let currentId = 0; From 6334b15e8eb121912eab0877012fec4685e20d33 Mon Sep 17 00:00:00 2001 From: rNoz Date: Fri, 31 Jul 2026 21:29:21 +0200 Subject: [PATCH 2/5] fix(ui): protect code/HTML/footnotes and quadratic risk in reference-link resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner review round for reference-style link resolution (#923): - Fence detection now mirrors the block parser's own naive rule exactly (full .trim() + startsWith('```'), any indentation, backtick-only — no ~~~ support) instead of a looser 0-3-space approximation, so indented and list-nested fences the block parser treats as code can never be rewritten. Aligns tilde-fence behavior the same way: since the block parser has no ~~~ support, the resolver no longer protects ~~~ blocks either. - Raw HTML blocks (
,
, etc.) are now protected using the
  same HTML_BLOCK_TAGS/HTML_BLOCK_OPEN_RE/VOID_HTML_TAGS the block
  parser itself uses, with the same three termination rules
  (blank-line, void single-line, balanced-depth).
- GFM footnote definitions ([^label]: ...) are excluded from
  collection entirely, so they and their [^label] references are
  never rewritten into inline links.
- A definition-shaped line is now only blanked when its label was
  actually consumed by a resolved reference outside a protected
  region. Unused definitions, and definitions referenced only from
  inside code/HTML, stay visible. This also fixes a plan-diff bug: a
  URL-only edit to a definition line used to blank to nothing on both
  sides of the diff (a real change rendering as empty); now the
  isolated diff chunk keeps the definition visible and diffs normally.
- CRLF lines are now recognized (definition regex tolerates a
  trailing \r) and preserved (a blanked line keeps its own \r).
- Bound the label/text capture groups (999 chars, CommonMark's own
  label limit) and the code-span alternative (5000 chars) so a long
  run of unmatched brackets/backticks can no longer cause quadratic
  backtracking within the 2MB annotate cap; added a defense-in-depth
  cap on the number of definitions tracked per document.
- Added coverage for nested brackets, backslash-escaped brackets,
  parenthesized destinations, idempotence, and confirmed dangerous
  destinations still flow through the existing sanitizeLinkUrl path
  unchanged.

Added a migration-caveat note to the existing annotation-anchor
section of packages/ui/HANDOFF.md: documents using reference-style
links render differently now, which can shift position-based anchors
captured before a host upgrades past this change.
---
 packages/ui/HANDOFF.md                   |   2 +
 packages/ui/utils/parser.test.ts         | 207 +++++++++++++++++++++-
 packages/ui/utils/parser.ts              | 209 +++++++++++++++++------
 packages/ui/utils/planDiffEngine.test.ts |  41 +++++
 4 files changed, 401 insertions(+), 58 deletions(-)

diff --git a/packages/ui/HANDOFF.md b/packages/ui/HANDOFF.md
index c39e031ed..b2bdf6c9c 100644
--- a/packages/ui/HANDOFF.md
+++ b/packages/ui/HANDOFF.md
@@ -267,6 +267,8 @@ interface Annotation {
 
 **Honesty note:** the failure path (step 4) is exercised in real use but is **not covered by automated tests** — nothing in the suite asserts the stale-anchor behavior. Treat the described degradation as accurate-but-unverified-by-CI, and test it in your integration if you depend on it.
 
+**Migration caveat — reference-style link resolution (#923):** `parseMarkdownToBlocks` now rewrites CommonMark reference links (`[text][id]`) and blanks their `[id]: url` definitions before splitting into blocks, so documents containing that syntax render differently than they did before this pass existed — a `[text][id]` pair that used to render as literal bracket text now renders as a link, and the definition line disappears from the rendered DOM entirely. That changes both the text and the per-tag DOM index at the affected positions. Any annotation whose `startMeta`/`endMeta` was captured against the *old* (pre-resolution) render of such a document — i.e. persisted before a host upgrades past this change — can restore onto the wrong text after upgrading, same as any other DOM-structure change described above; the text-search fallback (step 3) is the recovery path, and `originalText` is what to fall back to if you need to re-anchor server-side.
+
 ---
 
 ## Known rough edges (and why they're fine for now)
diff --git a/packages/ui/utils/parser.test.ts b/packages/ui/utils/parser.test.ts
index b1e6341b3..6a3f40122 100644
--- a/packages/ui/utils/parser.test.ts
+++ b/packages/ui/utils/parser.test.ts
@@ -56,24 +56,32 @@ describe("resolveReferenceLinks (#923)", () => {
       "array [0] and [TODO] here",
     );
     // A shortcut that does not name a definition stays literal even when other
-    // definitions exist.
+    // definitions exist. The definition itself is never consumed by anything
+    // in this document, so it stays visible too (PR #1168: unused definitions
+    // are not blanked).
     expect(resolveReferenceLinks("[TODO] and [0]\n\n[id]: https://e.com")).toBe(
-      "[TODO] and [0]\n\n",
+      "[TODO] and [0]\n\n[id]: https://e.com",
     );
   });
 
   test("does not double-link an inline link whose text matches a definition", () => {
+    // The inline link uses its own explicit URL and never consumes the
+    // definition, so the definition stays visible (PR #1168).
     expect(
       resolveReferenceLinks("[text](https://real.com)\n\n[text]: https://def.com"),
-    ).toBe("[text](https://real.com)\n\n");
+    ).toBe("[text](https://real.com)\n\n[text]: https://def.com");
   });
 
   test("never rewrites references inside fenced code blocks or inline code spans", () => {
+    // Both definitions are only ever "referenced" from inside a protected
+    // region (a fence, an inline code span), so neither reference resolves
+    // and neither definition is consumed — both stay visible verbatim
+    // (PR #1168).
     expect(resolveReferenceLinks("```\n[a][b]\n```\n\n[b]: https://e.com")).toBe(
-      "```\n[a][b]\n```\n\n",
+      "```\n[a][b]\n```\n\n[b]: https://e.com",
     );
     expect(resolveReferenceLinks("use `[a][b]` here\n\n[b]: https://e.com")).toBe(
-      "use `[a][b]` here\n\n",
+      "use `[a][b]` here\n\n[b]: https://e.com",
     );
   });
 
@@ -129,10 +137,15 @@ describe("resolveReferenceLinks (#923)", () => {
   });
 
   test("does not clobber a checked task-list item when an [x] definition exists", () => {
+    // The checkbox guard means the task-list `[x]` never resolves as a
+    // reference, so the "x" definition is never consumed and stays visible
+    // (PR #1168).
     expect(resolveReferenceLinks("- [x] done task\n- [ ] todo\n\n[x]: https://e.com")).toBe(
-      "- [x] done task\n- [ ] todo\n\n",
+      "- [x] done task\n- [ ] todo\n\n[x]: https://e.com",
+    );
+    expect(resolveReferenceLinks("1. [x] done\n\n[x]: https://e.com")).toBe(
+      "1. [x] done\n\n[x]: https://e.com",
     );
-    expect(resolveReferenceLinks("1. [x] done\n\n[x]: https://e.com")).toBe("1. [x] done\n\n");
   });
 
   test("resolves the shortcut image form", () => {
@@ -140,6 +153,186 @@ describe("resolveReferenceLinks (#923)", () => {
   });
 });
 
+describe("resolveReferenceLinks — owner review fixups (PR #1168)", () => {
+  test("does not rewrite a reference inside a fence indented 4+ spaces (block parser still treats it as code)", () => {
+    // The block parser detects a fence via `trimmed.startsWith('```')` after a
+    // full `.trim()` — ANY indentation still opens a code block. The resolver
+    // must recognize the exact same fence, not just fences within 3 spaces.
+    const md = "    ```\n    [a][b]\n    ```\n\n[b]: https://e.com";
+    // The code content must survive verbatim, and since "b" is never consumed
+    // outside the code fence, the definition itself must remain visible too —
+    // as its own trailing paragraph, not silently dropped.
+    expect(parseMarkdownToBlocks(md).map((b) => b.type)).toEqual(["code", "paragraph"]);
+    expect(resolveReferenceLinks(md)).toBe(md);
+  });
+
+  test("does not rewrite a reference inside a fence nested inside a list item at 4+ spaces", () => {
+    const md = "- outer\n  - inner\n    ```\n    [a][b]\n    ```\n\n[b]: https://e.com";
+    expect(resolveReferenceLinks(md)).toBe(md);
+  });
+
+  test("protects a link definition sitting inside a 
raw HTML block", () => { + const md = "
\nNotes\n\n[id]: https://from-details.com\n\n
"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("protects references and definitions inside a
 raw HTML block", () => {
+    const md = "
\n[a][b]\n
\n\n[b]: https://e.com"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("a reference used only inside a
block never counts as consumed, so the definition stays visible", () => { + const md = "
\n\n[x][id]\n\n
\n\n[id]: https://e.com"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("does not treat a GFM footnote definition ([^label]: ...) as a link reference definition", () => { + // A footnote body that is a bare token (looks exactly like a definition + // destination) is the real hazard — prose bodies with spaces already fail + // the destination shape by accident. + const md = "See the note.[^1]\n\n[^1]: https://example.com/footnote"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("does not clobber a footnote reference ([^1]) that happens to share a label with a real definition", () => { + const md = "See[^1] and [x][1]\n\n[^1]: https://footnote.com\n\n[1]: https://real-def.com"; + expect(resolveReferenceLinks(md)).toBe( + "See[^1] and [x](https://real-def.com)\n\n[^1]: https://footnote.com\n\n", + ); + }); + + test("leaves an entirely unused link reference definition visible", () => { + expect(resolveReferenceLinks("[id]: https://e.com\n\nSome unrelated text.")).toBe( + "[id]: https://e.com\n\nSome unrelated text.", + ); + }); + + test("leaves a definition visible when its only reference sits inside a fenced code block", () => { + const md = "```\n[x][id]\n```\n\n[id]: https://e.com"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("still blanks every definition line for a label once it is genuinely consumed, including redefinitions", () => { + expect( + resolveReferenceLinks("[id]: https://one.com\n[id]: https://two.com\n\n[x][id]"), + ).toBe("\n\n\n[x](https://one.com)"); + }); + + test("preserves total line count for a document mixing consumed, unused, and protected definitions", () => { + const md = [ + "# Heading", + "", + "[used]: https://used.com", + "[unused]: https://unused.com", + "", + "text [x][used]", + "", + "```", + "[y][coded]", + "```", + "", + "[coded]: https://coded.com", + ].join("\n"); + const resolved = resolveReferenceLinks(md); + expect(resolved.split("\n").length).toBe(md.split("\n").length); + const blocks = parseMarkdownToBlocks(md); + expect(blocks.find((b) => b.type === "heading")?.startLine).toBe(1); + // "unused" and "coded" (only referenced inside the fence) must remain + // visible; only the genuinely consumed "used" definition is blanked. + expect(resolved).toContain("[unused]: https://unused.com"); + expect(resolved).toContain("[coded]: https://coded.com"); + expect(resolved).not.toContain("[used]: https://used.com"); + expect(resolved).toContain("text [x](https://used.com)"); + }); + + test("supports CRLF documents and preserves the CRLF line endings", () => { + const md = "[text][id]\r\n\r\n[id]: https://e.com\r\n"; + expect(resolveReferenceLinks(md)).toBe("[text](https://e.com)\r\n\r\n\r\n"); + }); + + test("leaves an unconsumed CRLF definition visible with its line ending intact", () => { + const md = "[id]: https://e.com\r\n\r\nunrelated text\r\n"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("does not exhibit quadratic slowdown on a long run of unmatched '[' characters", () => { + const junk = "[".repeat(200_000); + const md = `${junk}\n\n[id]: https://e.com`; + const start = performance.now(); + const result = resolveReferenceLinks(md); + const elapsed = performance.now() - start; + // A naive unbounded backtracking scan would take many seconds to minutes + // here; a bounded one-pass scan finishes in well under a second. + expect(elapsed).toBeLessThan(1500); + expect(result.startsWith(junk)).toBe(true); + }); + + test("caps reference/definition label length so a single pathological label cannot force backtracking", () => { + const longLabel = "x".repeat(1500); + const md = `[text][${longLabel}]\n\n[${longLabel}]: https://e.com`; + // Deliberate safe degradation: a label above the bound is not resolved + // and its definition-shaped line is not collected either, so both sides + // are left untouched rather than partially/incorrectly rewritten. + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("is idempotent: resolving an already-resolved document is a no-op", () => { + const inputs = [ + "[text][id]\n\n[id]: https://e.com", + "![alt][id]\n\n[id]: /img.png", + "```\n[a][b]\n```\n\n[b]: https://e.com", + "- [x] done\n\n[x]: https://e.com", + "
\n[a][b]\n
\n\n[b]: https://e.com", + "[id]: https://e.com\n\nunused elsewhere", + "See[^1]\n\n[^1]: https://footnote.com", + ]; + for (const md of inputs) { + const once = resolveReferenceLinks(md); + const twice = resolveReferenceLinks(once); + expect(twice).toBe(once); + } + }); + + test("aligns tilde-fence handling with the block parser (neither treats ~~~ as a code fence)", () => { + const md = "~~~\n[a][b]\n~~~\n\n[b]: https://e.com"; + expect(parseMarkdownToBlocks(md).some((b) => b.type === "code")).toBe(false); + expect(resolveReferenceLinks(md)).toBe("~~~\n[a](https://e.com)\n~~~\n\n"); + }); + + test("resolves a destination containing a closing parenthesis", () => { + expect(resolveReferenceLinks("[x][id]\n\n[id]: https://e.com/a(b)")).toBe( + "[x](https://e.com/a(b))\n\n", + ); + }); + + test("backslash-escaped brackets never resolve, and their captured (unusable) label leaves the definition visible", () => { + const md = "Not a ref: \\[text\\]\\[id\\]\n\n[id]: https://e.com"; + expect(resolveReferenceLinks(md)).toBe(md); + }); + + test("a genuinely nested-bracket shortcut does not corrupt the destination pipeline or crash", () => { + // Unescaped nested brackets in link text are not legal CommonMark; the + // simplified single-pass scanner does not fully recover the intended + // reference, but it must never throw and must never do something that + // could bypass URL sanitization later. + const md = "[outer [inner] text][id]\n\n[id]: https://e.com"; + expect(() => resolveReferenceLinks(md)).not.toThrow(); + const result = resolveReferenceLinks(md); + expect(result).toContain("https://e.com"); + }); + + test("dangerous destinations still go through sanitizeLinkUrl identically to a hand-written inline link", () => { + // resolveReferenceLinks only ever emits `[text](dest)`, so it reuses the + // exact same inline-link rendering/sanitization path — it must never + // special-case or bypass it. + const resolved = resolveReferenceLinks("[x][id]\n\n[id]: javascript:alert(1)"); + expect(resolved).toBe("[x](javascript:alert(1))\n\n"); + // The literal string is unchanged (dangerous-protocol stripping happens + // downstream in sanitizeLinkUrl at render time), confirming this pass + // does not attempt — and therefore cannot get wrong — its own filtering. + }); +}); + describe("parseMarkdownToBlocks — code fences", () => { /** * Baseline: the common triple-backtick fence still works after the nested- diff --git a/packages/ui/utils/parser.ts b/packages/ui/utils/parser.ts index c221dc179..3add10648 100644 --- a/packages/ui/utils/parser.ts +++ b/packages/ui/utils/parser.ts @@ -204,13 +204,36 @@ export interface ParseMarkdownOptions { frontmatter?: boolean; } +// CommonMark bounds a link label to 999 characters. Reusing that bound here +// also caps the worst-case backtracking cost of the bracket-matching groups +// below to a constant per starting position, turning a document with a very +// long run of unmatched `[` characters (a real hazard within the 2MB annotate +// cap) into a linear scan instead of a quadratic one. A label longer than +// this is a deliberate, documented degradation: it is neither collected as a +// definition nor resolved as a reference, so it is simply left untouched +// rather than partially or incorrectly rewritten. +const MAX_REF_LABEL_CHARS = 999; +// Same reasoning applied to the inline-code-span alternative: bounding how far +// a lazy scan for a closing backtick run can travel keeps a line with many +// stray, unterminated backticks linear too. 5000 is far beyond any realistic +// inline code span, so legitimate spans are unaffected. +const MAX_CODE_SPAN_CHARS = 5000; +// Defense-in-depth cap on the number of definitions collected from a single +// document. A pathological document could otherwise grow the map without +// bound; this keeps that growth bounded even though ordinary documents never +// approach it. +const MAX_TRACKED_DEFINITIONS = 20_000; + // A link reference definition: `[label]: destination "optional title"`, with up // to three leading spaces. The destination is a bare token or an <...> form; any // trailing text must be a quoted or parenthesized title, otherwise the line is // ordinary prose (so `[Reminder]: call the bank` is NOT a definition). Matches -// the CommonMark shape closely enough for the simplified parser. -const REFERENCE_DEFINITION_RE = - /^ {0,3}\[([^\]]+)\]:[ \t]*(?:<([^>]*)>|(\S+))[ \t]*(?:"[^"]*"|'[^']*'|\([^)]*\))?[ \t]*$/; +// the CommonMark shape closely enough for the simplified parser. `\r?` before +// the end anchor tolerates a CRLF source (lines are split on `\n` only, so a +// CRLF line keeps its trailing `\r`). +const REFERENCE_DEFINITION_RE = new RegExp( + `^ {0,3}\\[([^\\]]{1,${MAX_REF_LABEL_CHARS}})\\]:[ \\t]*(?:<([^>]*)>|(\\S+))[ \\t]*(?:"[^"]*"|'[^']*'|\\([^)]*\\))?[ \\t]*\\r?$`, +); // One left-to-right pass over a line. The first alternative matches a whole // inline code span (balanced backtick run) so its contents are skipped; the @@ -219,7 +242,10 @@ const REFERENCE_DEFINITION_RE = // A bare `[text]` is the shortcut form, resolved only when it names a definition // and is not actually an inline link. Groups: 1 code ticks, 2 `!`, 3 text, // 4 second bracket, 5 label. -const REFERENCE_LINK_RE = /(`+).+?\1|(!?)\[([^\]]+)\](\[([^\]]*)\])?/g; +const REFERENCE_LINK_RE = new RegExp( + `(\`+)[^\\n]{0,${MAX_CODE_SPAN_CHARS}}?\\1|(!?)\\[([^\\]]{1,${MAX_REF_LABEL_CHARS}})\\](\\[([^\\]]{0,${MAX_REF_LABEL_CHARS}})\\])?`, + 'g', +); // CommonMark label matching is case-insensitive and collapses internal runs of // whitespace. @@ -227,41 +253,92 @@ const normalizeRefLabel = (label: string): string => label.trim().replace(/\s+/g, ' ').toLowerCase(); /** - * Mark every line that sits inside a fenced code block (opener, content, and - * closer), so link reference definitions and links inside code are left - * untouched. Mirrors the variable-length ``` / ~~~ fence handling in the block - * loop below. + * One pass over the lines that marks every line the block parser (below) will + * render as code or raw HTML — fenced code blocks and HTML blocks — so link + * reference definitions and references inside them are left completely + * untouched. This reuses the exact same conditions the block parser itself + * uses (not a looser approximation), so the two can never disagree about + * where code/HTML starts and ends: + * + * - Fences: `trimmed.startsWith('```')` after a full `.trim()` — the block + * parser has no minimum-indent exemption, so ANY indentation (a fence + * nested inside a list item, or simply indented 4+ spaces) still opens a + * code block, and this must too. Only backtick fences are recognized — + * the block parser has no `~~~` support, so this doesn't either (a `~~~` + * line is ordinary text to both). + * - Raw HTML blocks: the same `HTML_BLOCK_OPEN_RE`/`HTML_BLOCK_TAGS`/ + * `VOID_HTML_TAGS` the block parser uses, with the same three extents + * (blank-line termination for a leading close tag, single-line for void + * tags, balanced-depth scanning otherwise) — so a definition sitting + * inside `
` or `
` is protected exactly as + * far as the block parser's own HTML block extends. */ -const markFencedLines = (lines: string[]): boolean[] => { - const fenced = new Array(lines.length).fill(false); - let fenceChar = ''; - let fenceLen = 0; +const markProtectedLines = (lines: string[]): boolean[] => { + const isProtected = new Array(lines.length).fill(false); + let fenceLen = 0; // 0 = not currently inside a fence for (let i = 0; i < lines.length; i++) { - const trimmed = lines[i].replace(/^ {0,3}/, ''); - const opener = trimmed.match(/^(`{3,}|~{3,})/); - if (fenceLen === 0) { - if (opener) { - fenceChar = opener[1][0]; - fenceLen = opener[1].length; - fenced[i] = true; - } - } else { - fenced[i] = true; - const closer = new RegExp('^' + fenceChar + '{' + fenceLen + ',}[ \\t]*$'); - if (opener && opener[1][0] === fenceChar && closer.test(trimmed)) { - fenceChar = ''; - fenceLen = 0; + if (fenceLen > 0) { + isProtected[i] = true; + if (new RegExp('^\\s*`{' + fenceLen + ',}').test(lines[i])) fenceLen = 0; + continue; + } + const trimmed = lines[i].trim(); + if (trimmed.startsWith('```')) { + fenceLen = trimmed.match(/^`+/)![0].length; + isProtected[i] = true; + continue; + } + const htmlTagMatch = trimmed.match(HTML_BLOCK_OPEN_RE); + if (htmlTagMatch && HTML_BLOCK_TAGS.has(htmlTagMatch[1].toLowerCase())) { + const tagName = htmlTagMatch[1].toLowerCase(); + const isCloseTag = trimmed.startsWith('') && i + 1 < lines.length && lines[i + 1].trim() !== '') { + i++; + isProtected[i] = true; + } + } else { + const openRe = new RegExp(`<${tagName}(?:\\s|>|/|$)`, 'gi'); + const closeRe = new RegExp(``, 'gi'); + const depth = (lines[i].match(openRe) || []).length - (lines[i].match(closeRe) || []).length; + if (depth > 0) { + let j = i; + let d = depth; + const scanned: number[] = []; + while (d > 0 && j + 1 < lines.length) { + j++; + scanned.push(j); + d += (lines[j].match(openRe) || []).length; + d -= (lines[j].match(closeRe) || []).length; + } + if (d === 0) { + i = j; + for (const idx of scanned) isProtected[idx] = true; + } + } } } } - return fenced; + return isProtected; }; -/** Resolve reference links/images in one non-code line. A single left-to-right - * pass: an inline code span is matched as a whole and returned verbatim, so a - * reference-looking pattern inside backticks is never rewritten; only bracketed - * references outside code are resolved. */ -const resolveRefsInLine = (line: string, defs: Map): string => { +/** Resolve reference links/images in one non-code, non-HTML line. A single + * left-to-right pass: an inline code span is matched as a whole and returned + * verbatim, so a reference-looking pattern inside backticks is never + * rewritten; only bracketed references outside code are resolved. Every label + * that actually resolves against a definition is recorded into `usedLabels`, + * so the caller can tell a genuinely consumed definition from an unused one. */ +const resolveRefsInLine = ( + line: string, + defs: Map, + usedLabels: Set, +): string => { if (!line.includes('[')) return line; return line.replace( REFERENCE_LINK_RE, @@ -282,10 +359,13 @@ const resolveRefsInLine = (line: string, defs: Map): string => { } else { refLabel = label === '' ? text : label; } - const dest = defs.get(normalizeRefLabel(refLabel)); + const normalized = normalizeRefLabel(refLabel); + const dest = defs.get(normalized); // An unknown reference stays literal, matching CommonMark and avoiding // false links for bracketed prose like `[TODO]` or array indices. - return dest ? `${bang}[${text}](${dest})` : match; + if (!dest) return match; + usedLabels.add(normalized); + return `${bang}[${text}](${dest})`; }, ); }; @@ -294,37 +374,52 @@ const resolveRefsInLine = (line: string, defs: Map): string => { * Resolve CommonMark link reference definitions and reference links into inline * `[text](url)` links, so the shared inline renderer draws them instead of * showing raw `[text][id]` and `[id]: url` text (issue #923). Definitions and - * references inside fenced code blocks and inline code spans are left untouched. - * Definition lines are blanked in place rather than removed so block start-line - * numbers stay accurate. No-op (returns the input) when the document defines no - * references. + * references inside fenced code blocks, raw HTML blocks, and inline code spans + * are left untouched. A definition-shaped line is only ever blanked when its + * label was actually consumed by a resolved reference outside a protected + * region — an unused definition, or one referenced only from inside code/HTML, + * stays visible exactly as written. Blanked lines keep block start-line + * numbers accurate (and their own CRLF ending, so line endings round-trip). + * GFM footnote definitions (`[^label]: ...`) are never treated as link + * definitions. No-op (returns the input) when the document defines no + * (non-footnote) references. */ export const resolveReferenceLinks = (markdown: string): string => { if (!markdown.includes('[')) return markdown; const lines = markdown.split('\n'); - const fenced = markFencedLines(lines); + const isProtected = markProtectedLines(lines); const defs = new Map(); - const isDefLine = new Array(lines.length).fill(false); + // The normalized label a definition-shaped line defines, or null if the + // line isn't a definition (or is a footnote definition, which is never + // collected/blanked). + const defLabelByLine = new Array(lines.length).fill(null); // A definition cannot interrupt a paragraph (CommonMark 4.7): a line matching // the definition shape is only a definition when it can start a block, i.e. - // the previous line is the document start, blank, a fenced-code line (a code - // block is its own block), or itself a definition. Otherwise the line is - // paragraph continuation text and must be left untouched, or a bare + // the previous line is the document start, blank, a protected code/HTML + // line (each is its own block), or itself a definition. Otherwise the line + // is paragraph continuation text and must be left untouched, or a bare // `[word]: token` under a sentence would be silently deleted. let canStartDefinition = true; for (let i = 0; i < lines.length; i++) { - if (fenced[i]) { + if (isProtected[i]) { canStartDefinition = true; continue; } const blank = lines[i].trim() === ''; const match = canStartDefinition && !blank ? lines[i].match(REFERENCE_DEFINITION_RE) : null; if (match) { - const label = normalizeRefLabel(match[1]); - const dest = match[2] !== undefined ? match[2] : match[3]; - // First definition wins, per CommonMark. - if (label && dest && !defs.has(label)) defs.set(label, dest); - isDefLine[i] = true; + const rawLabel = match[1]; + // GFM footnote definition ([^label]: ...) — not a link reference + // definition. Leave it out of `defs` entirely so it can never be + // collected, blanked, or accidentally satisfy a footnote reference's + // lookup; it stays block-starting like any other definition line. + if (!rawLabel.startsWith('^') && defs.size < MAX_TRACKED_DEFINITIONS) { + const label = normalizeRefLabel(rawLabel); + const dest = match[2] !== undefined ? match[2] : match[3]; + // First definition wins, per CommonMark. + if (label && dest && !defs.has(label)) defs.set(label, dest); + defLabelByLine[i] = label; + } // A run of definitions stays eligible; canStartDefinition remains true. } else { // Blank keeps a new block startable; any other non-definition line starts @@ -333,8 +428,20 @@ export const resolveReferenceLinks = (markdown: string): string => { } } if (defs.size === 0) return markdown; - return lines - .map((line, i) => (isDefLine[i] ? '' : fenced[i] ? line : resolveRefsInLine(line, defs))) + const usedLabels = new Set(); + // Resolve references first; definition-shaped lines are passed through + // unresolved (never fed to resolveRefsInLine) so a definition's own + // `[label]` can never be mistaken for a reference to itself. + const resolved = lines.map((line, i) => + isProtected[i] || defLabelByLine[i] !== null ? line : resolveRefsInLine(line, defs, usedLabels), + ); + return resolved + .map((line, i) => { + const label = defLabelByLine[i]; + if (label === null || !usedLabels.has(label)) return line; + // Blank in place, preserving this line's own CRLF ending if it had one. + return line.endsWith('\r') ? '\r' : ''; + }) .join('\n'); }; diff --git a/packages/ui/utils/planDiffEngine.test.ts b/packages/ui/utils/planDiffEngine.test.ts index bbab62997..68cf3ca0a 100644 --- a/packages/ui/utils/planDiffEngine.test.ts +++ b/packages/ui/utils/planDiffEngine.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { computePlanDiff, computeInlineDiff } from "./planDiffEngine"; +import { parseMarkdownToBlocks } from "./parser"; describe("computePlanDiff — block-level behavior", () => { test("pure unchanged produces a single unchanged block, no stats", () => { @@ -547,3 +548,43 @@ describe("computePlanDiff — modified blocks populate inlineTokens when qualifi } }); }); + +describe("computePlanDiff — reference-link definition edits (PR #1168)", () => { + test("a URL-only edit to a link reference definition renders as a real change, not an empty diff", () => { + // diffLines isolates just the definition line into its own hunk (the + // paragraph that actually uses [docs][ref] is unchanged and elsewhere). + // Before the resolver stopped blanking unconsumed definitions, resolving + // that isolated one-line chunk on its own (no co-located reference to + // consume the label) blanked it to nothing on both sides, so the diff + // silently rendered zero blocks for a real content change. + const oldText = + "Read the [docs][ref] for details.\n\n[ref]: https://old.example.com/docs\n"; + const newText = + "Read the [docs][ref] for details.\n\n[ref]: https://new.example.com/docs\n"; + const { blocks } = computePlanDiff(oldText, newText); + const modified = blocks.find((b) => b.type === "modified"); + expect(modified).toBeDefined(); + expect(modified!.content.trim().length).toBeGreaterThan(0); + expect(modified!.oldContent!.trim().length).toBeGreaterThan(0); + // The isolated chunk must render as at least one visible block on each + // side — this is exactly what PlanCleanDiffView's MarkdownChunk uses to + // draw the change; zero blocks here is the "non-empty edit renders as + // empty" bug. + expect(parseMarkdownToBlocks(modified!.oldContent!).length).toBeGreaterThan(0); + expect(parseMarkdownToBlocks(modified!.content).length).toBeGreaterThan(0); + // The isolated chunk resolves to a single paragraph on each side (no + // co-located reference to consume the definition), so it qualifies for + // word-level inline diffing and the URL change is actually visible. + expect(modified!.inlineTokens).toBeDefined(); + const removedText = modified!.inlineTokens! + .filter((t) => t.type === "removed") + .map((t) => t.value) + .join(""); + const addedText = modified!.inlineTokens! + .filter((t) => t.type === "added") + .map((t) => t.value) + .join(""); + expect(removedText).toContain("old"); + expect(addedText).toContain("new"); + }); +}); From 9440be06acd5cb74a544be2792f7e1bbbc65eaa3 Mon Sep 17 00:00:00 2001 From: rNoz Date: Fri, 31 Jul 2026 22:00:57 +0200 Subject: [PATCH 3/5] fix(ui): bound HTML-block extent scan to kill quadratic unclosed-opener case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit markProtectedLines and parseMarkdownToBlocks each independently scanned line-by-line from a multi-line HTML opener until its balanced open/close depth returned to zero, giving up only at end-of-document. That scan never advanced the outer index on failure, so a document with many consecutive unclosed openers (e.g. thousands of bare
lines with no
anywhere) made every one of them re-run the same O(N) tail scan — O(N^2) total, a real hazard well within the 2MB annotate cap. Extract the scan into one shared helper, findHtmlBlockEnd, used by both call sites so they can't drift apart: - closeExistsFromLine lazily builds (once per tag name, cached per document) a suffix array answering whether a closing tag exists at or after a given line, so an opener that can never close is rejected in O(1) instead of scanning to EOF. - MAX_HTML_BLOCK_SCAN_LINES bounds the residual case (a closing tag exists far away but depth never actually reaches zero before it) to a constant amount of work per start position — a documented, safe degradation: a block whose true close sits beyond the cap is treated as unclosed, identically to today's 'no close ever found' case. Added a failing-before-fix perf test (many unclosed
lines took ~2.3-3.4s and blew a 800ms bound; now ~12-15ms) for both parseMarkdownToBlocks and resolveReferenceLinks, plus a parity test proving a real
...
block stays intact and identically protected/parsed among thousands of decoy unclosed
lines. --- packages/ui/utils/parser.test.ts | 70 ++++++++++++++++ packages/ui/utils/parser.ts | 139 ++++++++++++++++++++++++------- 2 files changed, 178 insertions(+), 31 deletions(-) diff --git a/packages/ui/utils/parser.test.ts b/packages/ui/utils/parser.test.ts index 6a3f40122..e90ff15b9 100644 --- a/packages/ui/utils/parser.test.ts +++ b/packages/ui/utils/parser.test.ts @@ -1220,6 +1220,76 @@ describe("parseMarkdownToBlocks — raw HTML blocks", () => { }); }); +describe("parseMarkdownToBlocks / resolveReferenceLinks — unclosed-HTML-opener perf (PR #1168 follow-up)", () => { + // Root cause: the balanced open/close depth scan for a multi-line HTML + // block does not advance the outer index when it fails to find a closing + // tag — so a document with many consecutive unclosed openers (e.g. + // thousands of bare `
` lines with no `
` anywhere) makes EVERY + // one of them independently re-scan all the way to end-of-document. That + // is O(N^2) work for N such lines, a real DoS-shaped hazard well within + // the 2MB annotate cap. Both markProtectedLines (used by + // resolveReferenceLinks) and parseMarkdownToBlocks's own HTML-block + // section duplicate this exact scan, so both must be fixed. + const N = 8000; + // Generous bound: a linear/bounded fix finishes in well under 100ms for + // this input; the pre-fix O(N^2) scan takes multiple seconds (measured + // ~2.2s for N=8000 during triage). 800ms leaves large machine-variance + // headroom while still failing clearly against the quadratic behavior. + const TIME_BOUND_MS = 800; + + test("parseMarkdownToBlocks stays fast with many consecutive unclosed
openers", () => { + const md = Array.from({ length: N }, () => "
").join("\n"); + const start = performance.now(); + const blocks = parseMarkdownToBlocks(md); + const elapsed = performance.now() - start; + expect(blocks).toHaveLength(N); + expect(blocks.every((b) => b.type === "html")).toBe(true); + expect(elapsed).toBeLessThan(TIME_BOUND_MS); + }); + + test("resolveReferenceLinks stays fast with many consecutive unclosed
openers", () => { + const md = + Array.from({ length: N }, () => "
").join("\n") + + "\n\n[x][id]\n\n[id]: https://e.com"; + const start = performance.now(); + const result = resolveReferenceLinks(md); + const elapsed = performance.now() - start; + expect(result).toContain("[x](https://e.com)"); + expect(elapsed).toBeLessThan(TIME_BOUND_MS); + }); + + test("parity: a real
...
block stays intact and identically protected/parsed among thousands of unclosed
decoys", () => { + const decoyCount = 5000; + const decoys = Array.from({ length: decoyCount }, () => "
").join("\n"); + const md = + `${decoys}\n\n` + + "
\nNotes\n\n[id]: https://from-details.com\n\n
" + + "\n\nAfter."; + + const start = performance.now(); + const blocks = parseMarkdownToBlocks(md); + const resolved = resolveReferenceLinks(md); + const elapsed = performance.now() - start; + expect(elapsed).toBeLessThan(TIME_BOUND_MS); + + // parseMarkdownToBlocks: the real
block is captured whole, + // undisturbed by the decoys before it, followed by its own paragraph. + const detailsBlock = blocks.find((b) => b.type === "html" && b.content.startsWith("
")); + expect(detailsBlock).toBeDefined(); + expect(detailsBlock!.content).toBe( + "
\nNotes\n\n[id]: https://from-details.com\n\n
", + ); + const paragraph = blocks.find((b) => b.type === "paragraph" && b.content === "After."); + expect(paragraph).toBeDefined(); + + // resolveReferenceLinks: the SAME
span is protected — the + // definition inside it is never consumed by anything, so it stays + // visible verbatim, exactly mirroring the block parser's own boundary + // for this block (not corrupted, not partially rewritten). + expect(resolved).toContain("[id]: https://from-details.com"); + }); +}); + describe("computeListIndices", () => { test("all unordered → all null", () => { const blocks = [li(0, false), li(0, false), li(0, false)]; diff --git a/packages/ui/utils/parser.ts b/packages/ui/utils/parser.ts index 3add10648..d4e015913 100644 --- a/packages/ui/utils/parser.ts +++ b/packages/ui/utils/parser.ts @@ -273,9 +273,99 @@ const normalizeRefLabel = (label: string): string => * inside `
` or `
` is protected exactly as * far as the block parser's own HTML block extends. */ +// Generous bound on how many lines ahead a balanced open/close depth scan may +// look for a multi-line HTML block's closing tag. Real nested HTML blocks in +// a plan/review document are never remotely this long; this only exists to +// bound the residual pathological case handled below (a closing tag exists +// somewhere far away but the running depth never actually returns to zero +// before it) so that case stays a bounded, constant amount of work too. +const MAX_HTML_BLOCK_SCAN_LINES = 2000; + +/** + * Lazily builds, once per tag name, a suffix boolean array answering "does a + * closing tag for this tag name exist at or after line k" in O(1) per query + * after an O(N) one-time build. Caching this per tag name (not per opening + * line) is what turns a document with many consecutive unclosed openers of + * the SAME tag (e.g. thousands of bare `
` lines) linear: the very first + * `
` pays the one-time O(N) cost, and every subsequent `
` gets an + * O(1) answer instead of independently re-scanning to end-of-document. The + * cache must be created fresh per document (per `markProtectedLines`/ + * `parseMarkdownToBlocks` call) since it indexes into that specific `lines` + * array. + */ +function closeExistsFromLine( + lines: string[], + tagName: string, + cache: Map, +): boolean[] { + const cached = cache.get(tagName); + if (cached) return cached; + const closeRe = new RegExp(``, 'i'); + const arr = new Array(lines.length + 1).fill(false); + for (let k = lines.length - 1; k >= 0; k--) { + arr[k] = arr[k + 1] || closeRe.test(lines[k]); + } + cache.set(tagName, arr); + return arr; +} + +/** + * Shared, bounded/linear helper computing the last line index of a balanced + * open/close-tag HTML block that opens at `startIndex` with the given + * already-computed `depth` (the opening line's own open-tag count minus + * close-tag count). Used by both `markProtectedLines` (the resolver's + * protection pass) and `parseMarkdownToBlocks` (the block parser) so the two + * can never disagree about a multi-line HTML block's extent, and so the fix + * below lives in exactly one place instead of two copies drifting apart. + * + * Root cause fixed here: naively scanning line-by-line from `startIndex` + * until depth returns to zero (or giving up at end-of-document) is fine for + * ONE opener, but a document with many consecutive unclosed openers repeats + * that full scan from every single one of them — each of the N openers pays + * for the O(N) tail, an O(N^2) hazard well within the 2MB annotate cap (this + * is exactly what a document full of bare `
` lines with no `
` + * anywhere triggers). Fixed with two layers: + * + * 1. `closeExistsFromLine` rejects in O(1) when no closing tag for this tag + * name exists anywhere later in the document — the common pathological + * case (no close at all) never scans a single line. + * 2. `MAX_HTML_BLOCK_SCAN_LINES` bounds the residual case (a closing tag + * exists far away but depth never actually reaches zero before it) to a + * constant amount of work per start position. This is a deliberate, + * documented degradation: a block whose true close sits beyond the cap + * is treated as unclosed, exactly like today's "no close ever found" + * case — i.e. it degrades to just its opening line, never partially or + * incorrectly extended. + * + * Returns `startIndex` unchanged when the block never closes (depth <= 0, + * no close exists anywhere, or the cap is hit without depth reaching zero). + */ +function findHtmlBlockEnd( + lines: string[], + startIndex: number, + tagName: string, + depth: number, + closeCache: Map, +): number { + if (depth <= 0) return startIndex; + if (!closeExistsFromLine(lines, tagName, closeCache)[startIndex + 1]) return startIndex; + const openRe = new RegExp(`<${tagName}(?:\\s|>|/|$)`, 'gi'); + const closeRe = new RegExp(``, 'gi'); + const limit = Math.min(lines.length - 1, startIndex + MAX_HTML_BLOCK_SCAN_LINES); + let j = startIndex; + let d = depth; + while (d > 0 && j < limit) { + j++; + d += (lines[j].match(openRe) || []).length; + d -= (lines[j].match(closeRe) || []).length; + } + return d === 0 ? j : startIndex; +} + const markProtectedLines = (lines: string[]): boolean[] => { const isProtected = new Array(lines.length).fill(false); let fenceLen = 0; // 0 = not currently inside a fence + const closeCache = new Map(); for (let i = 0; i < lines.length; i++) { if (fenceLen > 0) { isProtected[i] = true; @@ -307,20 +397,10 @@ const markProtectedLines = (lines: string[]): boolean[] => { const openRe = new RegExp(`<${tagName}(?:\\s|>|/|$)`, 'gi'); const closeRe = new RegExp(``, 'gi'); const depth = (lines[i].match(openRe) || []).length - (lines[i].match(closeRe) || []).length; - if (depth > 0) { - let j = i; - let d = depth; - const scanned: number[] = []; - while (d > 0 && j + 1 < lines.length) { - j++; - scanned.push(j); - d += (lines[j].match(openRe) || []).length; - d -= (lines[j].match(closeRe) || []).length; - } - if (d === 0) { - i = j; - for (const idx of scanned) isProtected[idx] = true; - } + const end = findHtmlBlockEnd(lines, i, tagName, depth, closeCache); + if (end > i) { + for (let idx = i + 1; idx <= end; idx++) isProtected[idx] = true; + i = end; } } } @@ -462,6 +542,11 @@ export const parseMarkdownToBlocks = (markdown: string, options?: ParseMarkdownO const lines = cleanMarkdown.split('\n'); const blocks: Block[] = []; let currentId = 0; + // Cache for findHtmlBlockEnd's "does a close tag exist later" fast path — + // scoped per parse call (per document) and shared across every HTML-block + // opener encountered below, so a document with many consecutive unclosed + // openers of the same tag only pays its one-time O(N) build cost once. + const htmlCloseCache = new Map(); let buffer: string[] = []; let currentType: Block['type'] = 'paragraph'; @@ -852,23 +937,15 @@ export const parseMarkdownToBlocks = (markdown: string, options?: ParseMarkdownO const openRe = new RegExp(`<${tagName}(?:\\s|>|/|$)`, 'gi'); const closeRe = new RegExp(``, 'gi'); const depth = (line.match(openRe) || []).length - (line.match(closeRe) || []).length; - if (depth > 0) { - // Scan ahead for the matching close tag. If none is ever found — a - // self-closing