From 3bd451d574f0c41b9075b7297b0bb35a3d15aca9 Mon Sep 17 00:00:00 2001 From: Ian Clarke Date: Wed, 30 Sep 2026 20:24:50 -0500 Subject: [PATCH 1/7] fix(ui): harden markdown rendering against malformed input - Pin the `markdown` dependency to a patched build ([patch.crates-io], freenet/markdown-rs at a fixed rev), which also parses long documents in linear rather than quadratic time. - Render page content and the editor preview through one bounded path that shows text past its limits (size, nesting per line, block length, table size, reference count and expansion, HTML size) as escaped plain text. - Walk and drop markdown trees without recursion. Claude-Session: https://claude.ai/code/session_013fuenPkF3T7ZkeypDFSRmx --- Cargo.lock | 3 +- Cargo.toml | 11 + ui/src/components.rs | 1 + ui/src/components/editor.rs | 22 +- ui/src/components/markdown_render.rs | 680 +++++++++++++++++++++++++++ ui/src/components/page_view.rs | 13 +- 6 files changed, 712 insertions(+), 18 deletions(-) create mode 100644 ui/src/components/markdown_render.rs diff --git a/Cargo.lock b/Cargo.lock index a69ab84..2263901 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2155,8 +2155,7 @@ dependencies = [ [[package]] name = "markdown" version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5cab8f2cadc416a82d2e783a1946388b31654d391d1c7d92cc1f03e295b1deb" +source = "git+https://github.com/freenet/markdown-rs?rev=c0646ed72010008a3ea92bf52be43082a09bdd12#c0646ed72010008a3ea92bf52be43082a09bdd12" dependencies = [ "unicode-id", ] diff --git a/Cargo.toml b/Cargo.toml index 70801ec..02c4e75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,3 +43,14 @@ strip = true [profile.release.package."*"] opt-level = 'z' + +# markdown 1.0.0 (the latest release) plus parser fixes that upstream has not +# released (wooorm/markdown-rs GH-22, GH-23, GH-31, GH-79, #206, #263, #267, and +# one more), and an indexed edit map that makes parsing a long document linear +# rather than quadratic in its length. The same branch River pins, one commit +# later. Drop this only once a release contains all of them; +# ui/src/components/markdown_render.rs `malformed_markdown_renders_without_panicking` +# fails if a parser fix is missing. +# https://github.com/freenet/markdown-rs/commits/freenet/panic-fixes +[patch.crates-io] +markdown = { git = "https://github.com/freenet/markdown-rs", rev = "c0646ed72010008a3ea92bf52be43082a09bdd12" } diff --git a/ui/src/components.rs b/ui/src/components.rs index c627b9d..fb1998f 100644 --- a/ui/src/components.rs +++ b/ui/src/components.rs @@ -1,6 +1,7 @@ mod add_site_dialog; mod editor; pub(crate) mod export_key; +mod markdown_render; mod page_view; mod pages_sidebar; mod sites_sidebar; diff --git a/ui/src/components/editor.rs b/ui/src/components/editor.rs index 5e43c5d..aab8fe5 100644 --- a/ui/src/components/editor.rs +++ b/ui/src/components/editor.rs @@ -132,18 +132,16 @@ pub fn Editor() -> Element { let title = state::EDITOR_TITLE.read().clone(); let content = state::EDITOR_CONTENT.read().clone(); - let preview_html = markdown::to_html_with_options(&content, &markdown::Options::gfm()) - .unwrap_or_else(|_| markdown::to_html(&content)); - // Inject heading ids so in-page anchor links (`[Link](#heading)`) work - // in the live preview as well as in the rendered page view, and - // beautify any Freenet URLs the user has typed. Honor the same - // gateway-detection flag the rendered page view uses — passing - // `true` unconditionally would show a same-origin `/v1/contract/ - // web/...` path in `dx serve` previews where there's no gateway - // behind Delta to resolve it. - let preview_html = super::page_view::inject_heading_ids(&preview_html); - let preview_html = super::page_view::finalize_anchors( - &preview_html, + // Render through the same bounded path as the page view, which also + // injects heading ids so in-page anchor links (`[Link](#heading)`) work + // in the live preview, and beautifies any Freenet URLs the user has + // typed. Honor the same gateway-detection flag the rendered page view + // uses — passing `true` unconditionally would show a same-origin + // `/v1/contract/web/...` path in `dx serve` previews where there's no + // gateway behind Delta to resolve it. + let preview_html = super::markdown_render::render_page_html( + &content, + &content, super::page_view::behind_gateway(), super::page_view::own_contract_id().as_deref(), ); diff --git a/ui/src/components/markdown_render.rs b/ui/src/components/markdown_render.rs new file mode 100644 index 0000000..ab0b5d9 --- /dev/null +++ b/ui/src/components/markdown_render.rs @@ -0,0 +1,680 @@ +//! Markdown rendering for page content, for both the page view and the +//! editor's live preview. +//! +//! Page content is written by other people, so the work spent rendering it is +//! bounded: text past the limits below is shown as escaped plain text instead +//! of markdown. The `markdown` crate is pinned to a patched build in the root +//! Cargo.toml (`[patch.crates-io]`). +//! +//! Adapted from River's message rendering (`ui/src/components/conversation.rs` +//! in freenet/river), with limits sized for long documents rather than chat +//! messages. + +use super::page_view::{finalize_anchors, inject_heading_ids}; + +/// Longest text rendered as markdown. +/// +/// 32 times River's message limit: long enough for a long document (the +/// largest markdown file across the Freenet and mediator repositories is +/// under 60 KiB). Ordinary text this long renders in about 0.1 s; the +/// slowest text found that passes every limit here takes about 0.7 s +/// natively and 1 s in the browser. (The patched `markdown` build parses in +/// time linear in a document's length apart from the inline cost below; +/// the 1.0.0 release is quadratic, so this limit is only safe with it.) +const MARKDOWN_MAX_SOURCE_BYTES: usize = 128 * 1024; + +/// Most block containers (`>`, `-`, `1.` ...) opened at the start of one line. +const MARKDOWN_MAX_LINE_NESTING: usize = 16; + +/// Bound on the sum, over each run of non-blank lines, of the run's length +/// squared, in bytes squared. +/// +/// Inline parsing (emphasis, brackets, code spans, inline HTML) can take time +/// quadratic in the length of a paragraph, and a paragraph never spans a +/// blank line. This allows a single 12 KiB run, or 36 runs of 2 KiB, and so +/// on. The largest real document measured (a 40 KiB design document with a +/// 9 KiB run) uses 70% of it, and 99% of the markdown files measured use +/// under a quarter. +const MARKDOWN_MAX_BLOCK_COST: u64 = 12 * 1024 * 12 * 1024; + +/// Most table cells the HTML can contain. A table's body rows are padded +/// to the width of its delimiter row, so a wide delimiter row followed by +/// many short lines expands to columns times rows cells. +const MARKDOWN_MAX_TABLE_CELLS: usize = 64 * 1024; + +/// Bound on `]:` occurrences times `]` occurrences. Each closing bracket is +/// checked against every definition, so many of both is quadratic. +const MARKDOWN_MAX_REFERENCE_LOOKUPS: usize = 1024 * 1024; + +/// Longest HTML kept from rendering markdown. It also bounds the HTML that +/// references can expand to before rendering (see `references_are_bounded`). +const MARKDOWN_MAX_HTML_BYTES: usize = 4 * 1024 * 1024; + +/// Render page markdown to HTML, with heading ids and finalized anchors. +/// +/// `text` is what to render as markdown. `plain` is what to show, escaped, if +/// `text` is too costly to render: the content as the author wrote it, before +/// any Delta-specific rewriting. +pub(super) fn render_page_html( + text: &str, + plain: &str, + rewrite_freenet_hrefs: bool, + own_contract_id: Option<&str>, +) -> String { + let Some(html) = render_gfm_bounded(text) else { + return plain_text_to_html(plain); + }; + let html = inject_heading_ids(&html); + let html = finalize_anchors(&html, rewrite_freenet_hrefs, own_contract_id); + if html.len() > MARKDOWN_MAX_HTML_BYTES { + return plain_text_to_html(plain); + } + html +} + +/// GFM HTML for `text`, or `None` if it is too costly to render. +fn render_gfm_bounded(text: &str) -> Option { + let block_cost = markdown_parse_cost(text)?; + if !references_are_bounded(text, block_cost) { + return None; + } + // GFM without MDX never fails to parse. + let html = markdown::to_html_with_options(text, &markdown::Options::gfm()).ok()?; + (html.len() <= MARKDOWN_MAX_HTML_BYTES).then_some(html) +} + +/// Whether `text` is cheap enough to parse as markdown. See the limits above. +#[cfg(test)] +fn markdown_cost_is_bounded(text: &str) -> bool { + markdown_parse_cost(text).is_some() +} + +/// `text`'s block cost (see `MARKDOWN_MAX_BLOCK_COST`) if it is cheap enough +/// to parse as markdown, or `None`. +fn markdown_parse_cost(text: &str) -> Option { + if text.len() > MARKDOWN_MAX_SOURCE_BYTES { + return None; + } + let closing_brackets = text.matches(']').count(); + let definitions = text.matches("]:").count(); + if closing_brackets.saturating_mul(definitions) > MARKDOWN_MAX_REFERENCE_LOOKUPS { + return None; + } + // A byte order mark can hide a marker from the counts below but not from + // the parser, which skips a leading one. + let text: std::borrow::Cow = if text.contains('\u{feff}') { + text.replace('\u{feff}', "").into() + } else { + text.into() + }; + + let mut block_cost: u64 = 0; + let mut run_bytes: u64 = 0; + let mut table_cells: usize = 0; + let mut run_table_columns: usize = 0; + for line in markdown_lines(&text) { + if line_container_depth(line) > MARKDOWN_MAX_LINE_NESTING { + return None; + } + if is_blank_line(line) { + block_cost = block_cost.saturating_add(run_bytes * run_bytes); + run_bytes = 0; + run_table_columns = 0; + continue; + } + run_bytes += line.len() as u64 + 1; + if let Some(columns) = delimiter_row_columns(line) { + run_table_columns = run_table_columns.max(columns); + } + // Every later line of the run may be a row of a table this wide. + table_cells = table_cells.saturating_add(run_table_columns); + } + block_cost = block_cost.saturating_add(run_bytes * run_bytes); + (block_cost <= MARKDOWN_MAX_BLOCK_COST && table_cells <= MARKDOWN_MAX_TABLE_CELLS) + .then_some(block_cost) +} + +/// The lines of `text`, split at `\n`, `\r\n` or a lone `\r` as markdown does. +fn markdown_lines(text: &str) -> impl Iterator { + let mut rest = Some(text); + std::iter::from_fn(move || { + let current = rest?; + match current.find(['\n', '\r']) { + Some(end) => { + let after = if current[end..].starts_with("\r\n") { + end + 2 + } else { + end + 1 + }; + rest = Some(¤t[after..]); + Some(¤t[..end]) + } + None => { + rest = None; + Some(current) + } + } + }) +} + +/// A line of only spaces and tabs: the only kind of line that ends every +/// paragraph and table. (Other whitespace, such as a no-break space, does not +/// make a line blank.) +fn is_blank_line(line: &str) -> bool { + line.bytes().all(|b| b == b' ' || b == b'\t') +} + +/// If `line` could be a table's delimiter row (`| --- | :-: |`, possibly +/// inside block quotes), how many columns it gives the table: the cells +/// between pipes that contain a `-`. Every delimiter row is made of only +/// these characters and has a `-` in every cell, so a delimiter row is never +/// missed and its columns are never under-counted. +fn delimiter_row_columns(line: &str) -> Option { + if !line + .bytes() + .all(|b| matches!(b, b'|' | b'-' | b':' | b' ' | b'\t' | b'>')) + { + return None; + } + let columns = line.split('|').filter(|cell| cell.contains('-')).count(); + (columns > 0).then_some(columns) +} + +/// How many block quote, list item or footnote definition markers open at +/// the start of `line` (an over-count is fine: it only makes plain text more +/// likely). +fn line_container_depth(line: &str) -> usize { + let bytes = line.as_bytes(); + let mut i = 0; + let mut depth = 0; + loop { + while i < bytes.len() && (bytes[i] == b' ' || bytes[i] == b'\t') { + i += 1; + } + let marker_end = match bytes.get(i) { + Some(b'>') => i + 1, + // A GFM footnote definition, `[^label]: `. + Some(b'[') if bytes.get(i + 1) == Some(&b'^') => match line[i..].find("]:") { + Some(end) => { + depth += 1; + i += end + 2; + continue; + } + None => return depth, + }, + Some(b'-' | b'*' | b'+') => i + 1, + Some(b'0'..=b'9') => { + let mut j = i; + while j < bytes.len() && bytes[j].is_ascii_digit() { + j += 1; + } + match bytes.get(j) { + Some(b'.' | b')') => j + 1, + _ => return depth, + } + } + _ => return depth, + }; + // A list marker must be followed by whitespace or the line's end. + let is_quote = bytes[i] == b'>'; + if !is_quote && !matches!(bytes.get(marker_end), None | Some(b' ' | b'\t')) { + return depth; + } + depth += 1; + i = marker_end; + } +} + +/// Whether the HTML that link and image references expand to is bounded. +/// +/// Each reference copies its definition's destination and title into the +/// HTML, so a few long definitions used many times expand a page +/// quadratically. A cheap bound comes first: every reference ends in `]`, and +/// see `longest_possible_definition`. Only if that bound is too high is the +/// text parsed to count exactly, and since that parses it a second time, only +/// text within half of the size and block cost limits gets that far. +fn references_are_bounded(text: &str, block_cost: u64) -> bool { + /// Bytes of HTML per byte of a copied destination or title (`"` becomes + /// `"`), and per reference for the tags around it. + const PER_BYTE: usize = 6; + const PER_REFERENCE: usize = 64; + + let closing_brackets = text.matches(']').count(); + let longest_definition = longest_possible_definition(text); + let cheap_bound = closing_brackets.saturating_mul( + longest_definition + .saturating_mul(PER_BYTE) + .saturating_add(PER_REFERENCE), + ); + if cheap_bound <= MARKDOWN_MAX_HTML_BYTES { + return true; + } + if text.len() > MARKDOWN_MAX_SOURCE_BYTES / 2 || block_cost > MARKDOWN_MAX_BLOCK_COST / 2 { + return false; + } + + let Ok(tree) = markdown::to_mdast(text, &markdown::ParseOptions::gfm()) else { + return false; + }; + let expansion = reference_expansion(&tree, longest_definition, PER_BYTE, PER_REFERENCE); + drop_mdast(tree); + expansion <= MARKDOWN_MAX_HTML_BYTES +} + +/// An upper bound on the destination plus title of any one link reference +/// definition. +/// +/// A definition is `[label]:`, a destination on that line or the next, and +/// an optional title opened by `"`, `'` or `(` on the destination's line or +/// the next. So a `]:` with none of those three characters in the rest of +/// its line and the two lines after it has nothing to copy beyond those +/// lines. Otherwise the title may run on, but never past a blank line, so +/// the bound is the rest of the run of non-blank lines. +fn longest_possible_definition(text: &str) -> usize { + let lines: Vec<&str> = markdown_lines(text).collect(); + // Bytes from the start of each line to the end of its run, counting each + // line ending as two bytes (`\r\n`) so it never under-counts. + let mut run_tail = vec![0usize; lines.len()]; + let mut tail = 0; + for (i, line) in lines.iter().enumerate().rev() { + tail = if is_blank_line(line) { + 0 + } else { + tail + line.len() + 2 + }; + run_tail[i] = tail; + } + let mut longest = 0; + for (i, line) in lines.iter().enumerate() { + for (pos, _) in line.match_indices("]:") { + let rest = &line[pos + 2..]; + let next: Vec<&str> = lines.iter().skip(i + 1).take(2).copied().collect(); + let may_have_title = std::iter::once(rest) + .chain(next.iter().copied()) + .any(|l| l.contains(['"', '\'', '('])); + let bound = if may_have_title { + run_tail[i] - pos + } else { + rest.len() + next.iter().map(|l| l.len() + 2).sum::() + }; + longest = longest.max(bound); + } + } + longest +} + +/// Bytes of HTML the references in `tree` expand to (an over-estimate). +fn reference_expansion( + root: &markdown::mdast::Node, + fallback_definition: usize, + per_byte: usize, + per_reference: usize, +) -> usize { + use markdown::mdast::Node; + use std::collections::HashMap; + + let mut definitions: HashMap<&str, usize> = HashMap::new(); + let mut references: Vec<&str> = Vec::new(); + // Iterative, not recursive: nesting depth is chosen by whoever wrote the + // text, and a recursive walk of a deep tree overflows the stack. Pre-order, + // so definitions are seen in document order and the first one wins, as it + // does when rendering. + let mut stack = vec![root]; + while let Some(node) = stack.pop() { + match node { + Node::Definition(d) => { + let len = d.url.len() + d.title.as_ref().map_or(0, String::len); + definitions.entry(d.identifier.as_str()).or_insert(len); + } + Node::LinkReference(r) => references.push(r.identifier.as_str()), + Node::ImageReference(r) => references.push(r.identifier.as_str()), + _ => {} + } + if let Some(children) = node.children() { + stack.extend(children.iter().rev()); + } + } + references.iter().fold(0usize, |total, id| { + // Every reference in the tree resolved to a definition; if the two + // disagree on an identifier, assume the longest possible one. + let len = definitions.get(id).copied().unwrap_or(fallback_definition); + total.saturating_add(len.saturating_mul(per_byte).saturating_add(per_reference)) + }) +} + +/// Drop a markdown AST without recursing: the tree's own `Drop` recurses once +/// per nesting level, which overflows the stack on deeply nested text. +fn drop_mdast(root: markdown::mdast::Node) { + let mut stack = vec![root]; + while let Some(mut node) = stack.pop() { + if let Some(children) = node.children_mut() { + stack.append(children); + } + } +} + +/// Text not rendered as markdown: escaped, with each line break kept. +fn plain_text_to_html(text: &str) -> String { + let text = text.replace("\r\n", "\n").replace('\r', "\n"); + format!("

{}

", escape_html(&text).replace('\n', "
\n")) +} + +fn escape_html(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + c => out.push(c), + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + const OWN_ID: &str = "EqJ5YpEEV3XLqEvKWLQHFhGAac2qXzSUoE6k2zbdnXBr"; + + fn references_bounded(text: &str) -> bool { + references_are_bounded(text, markdown_parse_cost(text).expect("cheap to parse")) + } + + /// Both rendering paths (page view and editor preview share + /// `render_page_html`), with and without the gateway href rewrite. + fn render_every_way(text: &str) { + for (rewrite, own) in [(true, None), (false, Some(OWN_ID)), (true, Some(OWN_ID))] { + let _ = render_page_html(text, text, rewrite, own); + } + } + + /// Inputs the unpatched `markdown` crate panics on (see + /// `[patch.crates-io]` in the root Cargo.toml). + #[test] + fn malformed_markdown_renders_without_panicking() { + let inputs = [ + // A line ending inside a link title or reference label. + "[a](b \"x\ny\")", + "[a](b 'x\ny')", + "[a](b (x\ny))", + "![a](b \"x\ny\")", + "[a](b \"x \ny\")", + "[a](b \"x \r\ny\")", + "[x](/x \"> \n\")", + "[a][b\nc]\n\n[b c]: d", + "[][a \n]\n\n[a ]:\0", + // An email address in an image title that spans lines. + "![a](b \"c@d.com\ne\")", + // Setext underlines next to each other. + "=\n=\n=\na\n=", + "}\n-\n--\n]\n=", + // A list item ending in unclosed code or HTML, then another marker. + "1. ", " ", + "- ", "---", "| - |", "\\\n", "[ ]", "[x]", "\0", "[[", "]]", "\n\n", + ]; + let mut state: u64 = 0x9E37_79B9_7F4A_7C15; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for _ in 0..3_000 { + let len = 1 + (next() % 24) as usize; + let text: String = (0..len) + .map(|_| PIECES[(next() % PIECES.len() as u64) as usize]) + .collect(); + let rendered = std::panic::catch_unwind(|| render_every_way(&text)); + assert!(rendered.is_ok(), "rendering {text:?} panicked"); + } + } + + /// The AST helpers walk and drop a tree without recursion, so a deep tree + /// cannot overflow the stack. Runs on a small stack to leave a margin + /// below wasm's 1 MiB. + #[test] + fn deep_markdown_trees_are_walked_without_recursion() { + std::thread::Builder::new() + .stack_size(256 * 1024) + .spawn(|| { + let deep = format!("{}[a]\n\n[a]: b", ">".repeat(5_000)); + let tree = markdown::to_mdast(&deep, &markdown::ParseOptions::gfm()).unwrap(); + assert_eq!(reference_expansion(&tree, 0, 6, 64), 6 + 64); + drop_mdast(tree); + }) + .expect("spawn") + .join() + .expect("walking a deep markdown tree panicked"); + } + + /// The deepest nesting that the exact reference count accepts renders, on + /// a small stack. Nesting across lines needs growing indentation, which + /// the size limit bounds. + #[test] + fn deepest_allowed_nesting_renders_on_a_small_stack() { + let mut text = String::new(); + let mut depth = 0; + loop { + let line = format!("{}- [a]\n\n", " ".repeat(depth)); + // Half the size limit, the most the exact reference count allows. + if text.len() + line.len() + 6_100 > MARKDOWN_MAX_SOURCE_BYTES / 2 { + break; + } + text.push_str(&line); + depth += 1; + } + // A definition followed, in the same run, by a long paragraph, so the + // cheap reference bound fails and the tree is built and walked. + text.push_str(&format!("[a]: b\n{}\n", "word ".repeat(1_200))); + assert!(depth > 200, "{depth}"); + assert!(markdown_cost_is_bounded(&text)); + let cheap = text.matches(']').count() * (longest_possible_definition(&text) * 6 + 64); + assert!(cheap > MARKDOWN_MAX_HTML_BYTES, "{cheap}"); + std::thread::Builder::new() + .stack_size(256 * 1024) + .spawn(move || { + let html = render_page_html(&text, &text, true, None); + assert!(html.contains("
    "), "{}", &html[..200]); + }) + .expect("spawn") + .join() + .expect("rendering deep nesting panicked"); + } + + /// Deep block nesting on one line, a wide table, too much text, or too + /// much text without a blank line is shown as plain text, however the + /// line breaks are written. + #[test] + fn costly_markdown_is_shown_as_plain_text() { + let deep_list = format!("{}x", "- ".repeat(MARKDOWN_MAX_LINE_NESTING + 1)); + let deep_quote = format!("{}x", "> 1. ".repeat(MARKDOWN_MAX_LINE_NESTING)); + // Pads every following line to 300 columns. + let wide_table = format!( + "{}\n{}\n{}", + "|a".repeat(300), + "|-".repeat(300), + "x\n".repeat(400) + ); + let quoted_wide_table = format!( + "> {}\n> {}\n{}", + "|a".repeat(300), + "| :-: ".repeat(300), + "x\n".repeat(400) + ); + // One paragraph past the block cost, written with each kind of line + // ending. + let long_paragraph = "word ".repeat(4 * 1024); + let crlf_paragraph = "word\r\n".repeat(4 * 1024); + let cr_paragraph = "word\r".repeat(4 * 1024); + // Many definitions and many closing brackets. + let lookups = format!("{}\n\n{}", "[a]: b\n\n".repeat(2_000), "x] ".repeat(1_000)); + for costly in [ + deep_list.clone(), + format!("a\r{deep_list}"), + format!("a\r\n{deep_quote}"), + format!("[^a]: {deep_list}"), + format!("\u{feff}{deep_list}"), + wide_table, + quoted_wide_table, + long_paragraph, + crlf_paragraph, + cr_paragraph, + lookups, + "a\n\n".repeat(MARKDOWN_MAX_SOURCE_BYTES / 3 + 1), + ] { + let head = &costly[..costly.len().min(80)]; + assert!(!markdown_cost_is_bounded(&costly), "{head:?}"); + let html = render_page_html(&costly, &costly, true, None); + assert!(html.starts_with("

    "), "{head:?}"); + assert!(!html.contains("

  • ") && !html.contains(""), "{head:?}"); + } + for fine in [ + format!("{}x", "- ".repeat(MARKDOWN_MAX_LINE_NESTING)), + "> quote\n- item\n 1. nested".to_string(), + "| a | b |\n| - | - |\n| 1 | 2 |".to_string(), + "-1 and 2.5 and -x".to_string(), + // A long code block of shell pipelines is not a wide table. + format!("```\n{}```", "a | b | c | d | e | f | g | h\n".repeat(200)), + ] { + assert!(markdown_cost_is_bounded(&fine), "{fine:?}"); + } + assert_eq!(line_container_depth(" > - 1) * x"), 4); + assert_eq!(line_container_depth(">>> x"), 3); + assert_eq!(line_container_depth("-x"), 0); + assert_eq!(line_container_depth("2024. was"), 1); + assert_eq!(line_container_depth("[^a]: > - x"), 3); + assert_eq!(line_container_depth("[^a] x"), 0); + assert_eq!(delimiter_row_columns("| --- | :-: |"), Some(2)); + assert_eq!(delimiter_row_columns("> -|-"), Some(2)); + assert_eq!(delimiter_row_columns("| a |"), None); + assert_eq!(delimiter_row_columns("| | |"), None); + assert_eq!(delimiter_row_columns("---"), Some(1)); + assert_eq!(delimiter_row_columns("|-|-|-|-|"), Some(4)); + } + + /// Lines split the way markdown splits them: a `\r\n` is one line ending, + /// not a line ending and an empty (blank) line. + #[test] + fn line_endings_split_like_markdown() { + let lines: Vec<&str> = markdown_lines("a\r\nb\rc\n\nd").collect(); + assert_eq!(lines, ["a", "b", "c", "", "d"]); + assert_eq!(markdown_lines("").collect::>(), [""]); + assert!(!is_blank_line("\u{a0}")); + assert!(is_blank_line(" \t")); + } + + /// A long document of ordinary markdown, near the size limit, renders as + /// markdown. + #[test] + fn long_ordinary_document_renders_as_markdown() { + let section = "## A heading\n\n\ + A paragraph with **bold**, _emphasis_, `code`, a [link](https://example.com/a) \ + and a [reference link][r]. It goes on for a while, the way paragraphs in \ + long documents do, with more words and another [link](#a-heading).\n\n\ + - item one\n- item two\n 1. nested\n\n\ + | a | b | c |\n|---|:-:|--:|\n| 1 | 2 | 3 |\n| 4 | 5 | 6 |\n\n\ + ```\nfn main() { println!(\"hi\"); }\n```\n\n\ + > A quote.\n\n"; + let mut text = String::new(); + while text.len() + section.len() < MARKDOWN_MAX_SOURCE_BYTES - 64 { + text.push_str(section); + } + text.push_str("[r]: https://example.com/r \"A title\"\n"); + assert!(markdown_cost_is_bounded(&text)); + let html = render_page_html(&text, &text, true, None); + assert!(html.contains("

    "), "{}", &html[..300]); + assert!(html.contains("") && html.contains("")); + assert!(html.contains("href=\"https://example.com/r\" title=\"A title\"")); + } + + /// A definition with a long title used many times would expand to far + /// more HTML than the limit, so it is caught before rendering. + #[test] + fn references_that_expand_too_far_are_shown_as_plain_text() { + let text = format!( + "[a]: b '{}'\n\n{}", + "\"".repeat(10_000), + "[a]\n\n".repeat(10_000) + ); + assert!(markdown_cost_is_bounded(&text)); + assert!(!references_bounded(&text)); + let html = render_page_html(&text, &text, true, None); + assert!(html.starts_with("

    [a]: b"), "{}", &html[..100]); + assert!(html.len() < 8 * text.len()); + + // Many reference links to short titled definitions are fine, although + // the cheap bound alone would reject them (a title may run to the end + // of the run, and the definitions are one run). + let mut many = String::new(); + for i in 0..150 { + many.push_str(&format!("See [this][r{i}] and [that][r{i}].\n\n")); + } + for i in 0..150 { + many.push_str(&format!("[r{i}]: https://example.com/{i} \"Title {i}\"\n")); + } + assert!(markdown_cost_is_bounded(&many)); + let cheap = many.matches(']').count() * (longest_possible_definition(&many) * 6 + 64); + assert!(cheap > MARKDOWN_MAX_HTML_BYTES, "{cheap}"); + assert!(references_bounded(&many)); + let html = render_page_html(&many, &many, true, None); + assert!(html.contains("href=\"https://example.com/149\" title=\"Title 149\"")); + + // Untitled definitions have a tight cheap bound, so a long document of + // them needs no second parse. + let mut untitled = String::new(); + for i in 0..300 { + untitled.push_str(&format!("See [this][r{i}].\n\n")); + } + for i in 0..300 { + untitled.push_str(&format!("[r{i}]: https://example.com/{i}\n")); + } + assert!(longest_possible_definition(&untitled) < 100); + assert!(render_page_html(&untitled, &untitled, true, None).contains("/299\"")); + } + + /// Text not rendered as markdown is escaped, keeps its line breaks, and + /// shows the content as written (here, before page-link resolution). + #[test] + fn plain_text_is_escaped_with_line_breaks() { + let costly = format!("[[Page]] & \"x\"\r\n{}", "- ".repeat(40)); + let html = render_page_html(&costly, "[[Page]]\r\nsecond", true, None); + assert_eq!(html, "

    <b>[[Page]]</b>
    \nsecond

    "); + assert!(!render_page_html(&costly, &costly, true, None).contains("")); + } +} diff --git a/ui/src/components/page_view.rs b/ui/src/components/page_view.rs index 7bbfd42..c27d686 100644 --- a/ui/src/components/page_view.rs +++ b/ui/src/components/page_view.rs @@ -198,12 +198,17 @@ pub fn PageView() -> Element { /// (`[Link to Heading](#heading)`) work natively, and beautifying bare /// Freenet web-contract URLs (`http://gateway/v1/contract/web//...`) /// into `freenet:[/path]` labels with same-origin hrefs. +/// +/// Content too costly to render as markdown is shown as plain text (see +/// `markdown_render`). fn render_markdown(content: &str) -> String { let resolved = resolve_page_links(content); - let html = markdown::to_html_with_options(&resolved, &markdown::Options::gfm()) - .unwrap_or_else(|_| markdown::to_html(&resolved)); - let html = inject_heading_ids(&html); - finalize_anchors(&html, behind_gateway(), own_contract_id().as_deref()) + super::markdown_render::render_page_html( + &resolved, + content, + behind_gateway(), + own_contract_id().as_deref(), + ) } /// True when Delta is currently being served from a path under From deb9ffa39e1a47f2a7051be145b133697fdea76b Mon Sep 17 00:00:00 2001 From: Ian Clarke Date: Wed, 30 Sep 2026 21:02:35 -0500 Subject: [PATCH 2/7] fix(ui): tighten markdown rendering limits and pin them with tests - Skip only a single leading byte order mark when measuring, as the parser does; any other U+FEFF counts as ordinary text. - Add a whole-document limit on block container markers, alongside the per-line one. - Check the page size before resolving [[page links]], cap the resolved output and the work spent on it, stop scanning at the last closing bracket pair, and look titles up through an index. - Make the limits a struct so tests reach each one with tiny inputs, and assert the counters against them: exact boundaries, linear scaling, the exact reference count and its half-limit gate, both HTML caps. - Run the parser regression inputs through the syntax-tree path too, test that the page view and editor preview render through the bounded module, and pin the patched markdown build in Cargo.lock. [AI-assisted - Claude] Claude-Session: https://claude.ai/code/session_013fuenPkF3T7ZkeypDFSRmx --- ui/src/components/markdown_render.rs | 1036 +++++++++++++++++++------- ui/src/components/page_view.rs | 327 ++++++-- 2 files changed, 1039 insertions(+), 324 deletions(-) diff --git a/ui/src/components/markdown_render.rs b/ui/src/components/markdown_render.rs index ab0b5d9..8fbbb4f 100644 --- a/ui/src/components/markdown_render.rs +++ b/ui/src/components/markdown_render.rs @@ -6,49 +6,77 @@ //! of markdown. The `markdown` crate is pinned to a patched build in the root //! Cargo.toml (`[patch.crates-io]`). //! +//! This module is the only code that calls the `markdown` crate; a test in +//! it fails if anything else does, or if the page view or editor preview +//! stops rendering through it. +//! //! Adapted from River's message rendering (`ui/src/components/conversation.rs` //! in freenet/river), with limits sized for long documents rather than chat //! messages. use super::page_view::{finalize_anchors, inject_heading_ids}; -/// Longest text rendered as markdown. -/// -/// 32 times River's message limit: long enough for a long document (the -/// largest markdown file across the Freenet and mediator repositories is -/// under 60 KiB). Ordinary text this long renders in about 0.1 s; the -/// slowest text found that passes every limit here takes about 0.7 s -/// natively and 1 s in the browser. (The patched `markdown` build parses in -/// time linear in a document's length apart from the inline cost below; -/// the 1.0.0 release is quadratic, so this limit is only safe with it.) -const MARKDOWN_MAX_SOURCE_BYTES: usize = 128 * 1024; - -/// Most block containers (`>`, `-`, `1.` ...) opened at the start of one line. -const MARKDOWN_MAX_LINE_NESTING: usize = 16; - -/// Bound on the sum, over each run of non-blank lines, of the run's length -/// squared, in bytes squared. +/// Bounds on the work spent rendering one text as markdown. Text past any of +/// them is shown as plain text. The values used are `LIMITS`; tests use +/// smaller ones so their inputs stay tiny. +#[derive(Clone, Copy, Debug)] +struct Limits { + /// Longest text rendered as markdown, in bytes. + source_bytes: usize, + /// Most block containers (`>`, `-`, `1.` ...) opened at the start of one + /// line. + line_nesting: usize, + /// Most block container markers in the whole text: the sum over its lines + /// of the containers each opens (see `line_container_depth`). + container_markers: usize, + /// Bound on the sum, over each run of non-blank lines, of the run's + /// length squared, in bytes squared. + block_cost: u64, + /// Most table cells the HTML can contain. + table_cells: usize, + /// Bound on `]:` occurrences times `]` occurrences. + reference_lookups: usize, + /// Longest HTML kept from rendering markdown, in bytes. It also bounds + /// the HTML that references can expand to before rendering (see + /// `references_are_bounded`). + html_bytes: usize, +} + +/// The limits page content is rendered under. /// -/// Inline parsing (emphasis, brackets, code spans, inline HTML) can take time -/// quadratic in the length of a paragraph, and a paragraph never spans a -/// blank line. This allows a single 12 KiB run, or 36 runs of 2 KiB, and so -/// on. The largest real document measured (a 40 KiB design document with a -/// 9 KiB run) uses 70% of it, and 99% of the markdown files measured use -/// under a quarter. -const MARKDOWN_MAX_BLOCK_COST: u64 = 12 * 1024 * 12 * 1024; - -/// Most table cells the HTML can contain. A table's body rows are padded -/// to the width of its delimiter row, so a wide delimiter row followed by -/// many short lines expands to columns times rows cells. -const MARKDOWN_MAX_TABLE_CELLS: usize = 64 * 1024; - -/// Bound on `]:` occurrences times `]` occurrences. Each closing bracket is -/// checked against every definition, so many of both is quadratic. -const MARKDOWN_MAX_REFERENCE_LOOKUPS: usize = 1024 * 1024; - -/// Longest HTML kept from rendering markdown. It also bounds the HTML that -/// references can expand to before rendering (see `references_are_bounded`). -const MARKDOWN_MAX_HTML_BYTES: usize = 4 * 1024 * 1024; +/// - `source_bytes`: 32 times River's message limit, long enough for a long +/// document (the largest markdown file across the Freenet and mediator +/// repositories is under 60 KiB). Ordinary text this long renders in about +/// 0.1 s; the slowest text found that passes every limit takes about 0.7 s +/// natively and 1 s in the browser. (The patched `markdown` build parses +/// in time linear in a document's length apart from the inline cost below; +/// the 1.0.0 release is quadratic, so this limit is only safe with it.) +/// - `line_nesting`: the same as River. Parser time grows faster than +/// linearly with nesting. +/// - `container_markers`: the most any real markdown file measured opens is +/// about 700, so this leaves a wide margin. A very long checklist or list +/// (thousands of items) is shown as plain text. +/// - `block_cost`: inline parsing (emphasis, brackets, code spans, inline +/// HTML) can take time quadratic in the length of a paragraph, and a +/// paragraph never spans a blank line. This allows a single 12 KiB run, or +/// 36 runs of 2 KiB, and so on. The largest real document measured (a +/// 40 KiB design document with a 9 KiB run) uses 70% of it, and 99% of the +/// markdown files measured use under a quarter. +/// - `table_cells`: a table's body rows are padded to the width of its +/// delimiter row, so a wide delimiter row followed by many short lines +/// expands to columns times rows cells. +/// - `reference_lookups`: each closing bracket is checked against every +/// definition, so many of both is quadratic. +/// - `html_bytes`: a backstop on output size. +const LIMITS: Limits = Limits { + source_bytes: 128 * 1024, + line_nesting: 16, + container_markers: 4 * 1024, + block_cost: 12 * 1024 * 12 * 1024, + table_cells: 64 * 1024, + reference_lookups: 1024 * 1024, + html_bytes: 4 * 1024 * 1024, +}; /// Render page markdown to HTML, with heading ids and finalized anchors. /// @@ -61,77 +89,153 @@ pub(super) fn render_page_html( rewrite_freenet_hrefs: bool, own_contract_id: Option<&str>, ) -> String { - let Some(html) = render_gfm_bounded(text) else { + render_page_html_with(&LIMITS, text, plain, rewrite_freenet_hrefs, own_contract_id) +} + +/// Render a page's stored content for the page view. +/// +/// `resolve_links(content, max_len)` rewrites Delta's `[[...]]` page links +/// into markdown, or returns `None` if the result would be longer than +/// `max_len`. It is only called for content within the size limit, and its +/// result goes through the same limits as any other text. Content that is +/// too long, or too costly to render, is shown as written, as plain text. +pub(super) fn render_page_view_html( + content: &str, + resolve_links: impl FnOnce(&str, usize) -> Option, + rewrite_freenet_hrefs: bool, + own_contract_id: Option<&str>, +) -> String { + render_page_view_html_with( + &LIMITS, + content, + resolve_links, + rewrite_freenet_hrefs, + own_contract_id, + ) +} + +fn render_page_view_html_with( + limits: &Limits, + content: &str, + resolve_links: impl FnOnce(&str, usize) -> Option, + rewrite_freenet_hrefs: bool, + own_contract_id: Option<&str>, +) -> String { + // Checked before resolving links, which is work in proportion to the + // text. Resolved text longer than this would be plain text anyway. + if content.len() > limits.source_bytes { + return plain_text_to_html(content); + } + match resolve_links(content, limits.source_bytes) { + Some(resolved) => render_page_html_with( + limits, + &resolved, + content, + rewrite_freenet_hrefs, + own_contract_id, + ), + None => plain_text_to_html(content), + } +} + +fn render_page_html_with( + limits: &Limits, + text: &str, + plain: &str, + rewrite_freenet_hrefs: bool, + own_contract_id: Option<&str>, +) -> String { + let Some(html) = render_gfm_bounded(text, limits) else { return plain_text_to_html(plain); }; let html = inject_heading_ids(&html); let html = finalize_anchors(&html, rewrite_freenet_hrefs, own_contract_id); - if html.len() > MARKDOWN_MAX_HTML_BYTES { + if html.len() > limits.html_bytes { return plain_text_to_html(plain); } html } /// GFM HTML for `text`, or `None` if it is too costly to render. -fn render_gfm_bounded(text: &str) -> Option { - let block_cost = markdown_parse_cost(text)?; - if !references_are_bounded(text, block_cost) { +fn render_gfm_bounded(text: &str, limits: &Limits) -> Option { + let cost = markdown_parse_cost(text, limits)?; + if !references_are_bounded(text, &cost, limits) { return None; } // GFM without MDX never fails to parse. let html = markdown::to_html_with_options(text, &markdown::Options::gfm()).ok()?; - (html.len() <= MARKDOWN_MAX_HTML_BYTES).then_some(html) + (html.len() <= limits.html_bytes).then_some(html) } -/// Whether `text` is cheap enough to parse as markdown. See the limits above. -#[cfg(test)] -fn markdown_cost_is_bounded(text: &str) -> bool { - markdown_parse_cost(text).is_some() +/// The work a text would take to parse, as counted against `Limits`. +#[derive(Debug, Default, PartialEq)] +struct ParseCost { + /// See `Limits::block_cost`. + block: u64, + /// See `Limits::container_markers`. + container_markers: usize, + /// The most containers opened at the start of any one line. + deepest_line: usize, + /// See `Limits::table_cells`. + table_cells: usize, + /// `]:` occurrences times `]` occurrences. + reference_lookups: usize, } -/// `text`'s block cost (see `MARKDOWN_MAX_BLOCK_COST`) if it is cheap enough -/// to parse as markdown, or `None`. -fn markdown_parse_cost(text: &str) -> Option { - if text.len() > MARKDOWN_MAX_SOURCE_BYTES { - return None; +impl ParseCost { + /// Measure `text`, in time linear in its length. + fn of(text: &str) -> Self { + let closing_brackets = text.matches(']').count(); + let definitions = text.matches("]:").count(); + let mut cost = ParseCost { + reference_lookups: closing_brackets.saturating_mul(definitions), + ..ParseCost::default() + }; + // The parser skips one byte order mark at the very start, so a marker + // just after it still opens a container. Any other U+FEFF is ordinary + // text to the parser, and so it is here. + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + + let mut run_bytes: u64 = 0; + let mut run_table_columns: usize = 0; + for line in markdown_lines(text) { + let depth = line_container_depth(line); + cost.deepest_line = cost.deepest_line.max(depth); + cost.container_markers = cost.container_markers.saturating_add(depth); + if is_blank_line(line) { + cost.block = cost.block.saturating_add(run_bytes * run_bytes); + run_bytes = 0; + run_table_columns = 0; + continue; + } + run_bytes += line.len() as u64 + 1; + if let Some(columns) = delimiter_row_columns(line) { + run_table_columns = run_table_columns.max(columns); + } + // Every later line of the run may be a row of a table this wide. + cost.table_cells = cost.table_cells.saturating_add(run_table_columns); + } + cost.block = cost.block.saturating_add(run_bytes * run_bytes); + cost } - let closing_brackets = text.matches(']').count(); - let definitions = text.matches("]:").count(); - if closing_brackets.saturating_mul(definitions) > MARKDOWN_MAX_REFERENCE_LOOKUPS { - return None; + + fn is_within(&self, limits: &Limits) -> bool { + self.deepest_line <= limits.line_nesting + && self.container_markers <= limits.container_markers + && self.block <= limits.block_cost + && self.table_cells <= limits.table_cells + && self.reference_lookups <= limits.reference_lookups } - // A byte order mark can hide a marker from the counts below but not from - // the parser, which skips a leading one. - let text: std::borrow::Cow = if text.contains('\u{feff}') { - text.replace('\u{feff}', "").into() - } else { - text.into() - }; +} - let mut block_cost: u64 = 0; - let mut run_bytes: u64 = 0; - let mut table_cells: usize = 0; - let mut run_table_columns: usize = 0; - for line in markdown_lines(&text) { - if line_container_depth(line) > MARKDOWN_MAX_LINE_NESTING { - return None; - } - if is_blank_line(line) { - block_cost = block_cost.saturating_add(run_bytes * run_bytes); - run_bytes = 0; - run_table_columns = 0; - continue; - } - run_bytes += line.len() as u64 + 1; - if let Some(columns) = delimiter_row_columns(line) { - run_table_columns = run_table_columns.max(columns); - } - // Every later line of the run may be a row of a table this wide. - table_cells = table_cells.saturating_add(run_table_columns); +/// `text`'s parse cost if it is cheap enough to parse as markdown, or `None`. +/// The size limit is checked first, so measuring is bounded too. +fn markdown_parse_cost(text: &str, limits: &Limits) -> Option { + if text.len() > limits.source_bytes { + return None; } - block_cost = block_cost.saturating_add(run_bytes * run_bytes); - (block_cost <= MARKDOWN_MAX_BLOCK_COST && table_cells <= MARKDOWN_MAX_TABLE_CELLS) - .then_some(block_cost) + let cost = ParseCost::of(text); + cost.is_within(limits).then_some(cost) } /// The lines of `text`, split at `\n`, `\r\n` or a lone `\r` as markdown does. @@ -225,6 +329,12 @@ fn line_container_depth(line: &str) -> usize { } } +/// Bytes of HTML per byte of a copied destination or title (`"` becomes +/// `"`). +const REFERENCE_HTML_PER_BYTE: usize = 6; +/// Bytes of HTML per reference for the tags around it. +const REFERENCE_HTML_PER_REFERENCE: usize = 64; + /// Whether the HTML that link and image references expand to is bounded. /// /// Each reference copies its definition's destination and title into the @@ -233,32 +343,84 @@ fn line_container_depth(line: &str) -> usize { /// see `longest_possible_definition`. Only if that bound is too high is the /// text parsed to count exactly, and since that parses it a second time, only /// text within half of the size and block cost limits gets that far. -fn references_are_bounded(text: &str, block_cost: u64) -> bool { - /// Bytes of HTML per byte of a copied destination or title (`"` becomes - /// `"`), and per reference for the tags around it. - const PER_BYTE: usize = 6; - const PER_REFERENCE: usize = 64; - +fn references_are_bounded(text: &str, cost: &ParseCost, limits: &Limits) -> bool { let closing_brackets = text.matches(']').count(); let longest_definition = longest_possible_definition(text); let cheap_bound = closing_brackets.saturating_mul( longest_definition - .saturating_mul(PER_BYTE) - .saturating_add(PER_REFERENCE), + .saturating_mul(REFERENCE_HTML_PER_BYTE) + .saturating_add(REFERENCE_HTML_PER_REFERENCE), ); - if cheap_bound <= MARKDOWN_MAX_HTML_BYTES { + if cheap_bound <= limits.html_bytes { return true; } - if text.len() > MARKDOWN_MAX_SOURCE_BYTES / 2 || block_cost > MARKDOWN_MAX_BLOCK_COST / 2 { + if text.len() > limits.source_bytes / 2 || cost.block > limits.block_cost / 2 { return false; } + exact_reference_expansion(text, longest_definition) + .is_some_and(|expansion| expansion.html_bytes <= limits.html_bytes) +} - let Ok(tree) = markdown::to_mdast(text, &markdown::ParseOptions::gfm()) else { - return false; - }; - let expansion = reference_expansion(&tree, longest_definition, PER_BYTE, PER_REFERENCE); +/// What walking a syntax tree for references found. +#[derive(Debug, PartialEq)] +struct ReferenceExpansion { + /// Bytes of HTML the references expand to (an over-estimate). + html_bytes: usize, + /// Tree nodes visited. + nodes: usize, +} + +/// Parse `text` to a syntax tree and count what its references expand to, +/// then drop the tree. `None` only if the parser reports an error, which it +/// does not for GFM. +fn exact_reference_expansion(text: &str, longest_definition: usize) -> Option { + let tree = markdown::to_mdast(text, &markdown::ParseOptions::gfm()).ok()?; + let expansion = reference_expansion(&tree, longest_definition); drop_mdast(tree); - expansion <= MARKDOWN_MAX_HTML_BYTES + Some(expansion) +} + +/// Count what the references in `root` expand to, visiting each node once. +fn reference_expansion( + root: &markdown::mdast::Node, + fallback_definition: usize, +) -> ReferenceExpansion { + use markdown::mdast::Node; + use std::collections::HashMap; + + let mut definitions: HashMap<&str, usize> = HashMap::new(); + let mut references: Vec<&str> = Vec::new(); + let mut nodes = 0; + // Iterative, not recursive: nesting depth is chosen by whoever wrote the + // text, and a recursive walk of a deep tree overflows the stack. Pre-order, + // so definitions are seen in document order and the first one wins, as it + // does when rendering. + let mut stack = vec![root]; + while let Some(node) = stack.pop() { + nodes += 1; + match node { + Node::Definition(d) => { + let len = d.url.len() + d.title.as_ref().map_or(0, String::len); + definitions.entry(d.identifier.as_str()).or_insert(len); + } + Node::LinkReference(r) => references.push(r.identifier.as_str()), + Node::ImageReference(r) => references.push(r.identifier.as_str()), + _ => {} + } + if let Some(children) = node.children() { + stack.extend(children.iter().rev()); + } + } + let html_bytes = references.iter().fold(0usize, |total, id| { + // Every reference in the tree resolved to a definition; if the two + // disagree on an identifier, assume the longest possible one. + let len = definitions.get(id).copied().unwrap_or(fallback_definition); + total.saturating_add( + len.saturating_mul(REFERENCE_HTML_PER_BYTE) + .saturating_add(REFERENCE_HTML_PER_REFERENCE), + ) + }); + ReferenceExpansion { html_bytes, nodes } } /// An upper bound on the destination plus title of any one link reference @@ -303,45 +465,6 @@ fn longest_possible_definition(text: &str) -> usize { longest } -/// Bytes of HTML the references in `tree` expand to (an over-estimate). -fn reference_expansion( - root: &markdown::mdast::Node, - fallback_definition: usize, - per_byte: usize, - per_reference: usize, -) -> usize { - use markdown::mdast::Node; - use std::collections::HashMap; - - let mut definitions: HashMap<&str, usize> = HashMap::new(); - let mut references: Vec<&str> = Vec::new(); - // Iterative, not recursive: nesting depth is chosen by whoever wrote the - // text, and a recursive walk of a deep tree overflows the stack. Pre-order, - // so definitions are seen in document order and the first one wins, as it - // does when rendering. - let mut stack = vec![root]; - while let Some(node) = stack.pop() { - match node { - Node::Definition(d) => { - let len = d.url.len() + d.title.as_ref().map_or(0, String::len); - definitions.entry(d.identifier.as_str()).or_insert(len); - } - Node::LinkReference(r) => references.push(r.identifier.as_str()), - Node::ImageReference(r) => references.push(r.identifier.as_str()), - _ => {} - } - if let Some(children) = node.children() { - stack.extend(children.iter().rev()); - } - } - references.iter().fold(0usize, |total, id| { - // Every reference in the tree resolved to a definition; if the two - // disagree on an identifier, assume the longest possible one. - let len = definitions.get(id).copied().unwrap_or(fallback_definition); - total.saturating_add(len.saturating_mul(per_byte).saturating_add(per_reference)) - }) -} - /// Drop a markdown AST without recursing: the tree's own `Drop` recurses once /// per nesting level, which overflows the stack on deeply nested text. fn drop_mdast(root: markdown::mdast::Node) { @@ -380,8 +503,30 @@ mod tests { const OWN_ID: &str = "EqJ5YpEEV3XLqEvKWLQHFhGAac2qXzSUoE6k2zbdnXBr"; - fn references_bounded(text: &str) -> bool { - references_are_bounded(text, markdown_parse_cost(text).expect("cheap to parse")) + /// Small limits, so tests can reach each one with a tiny input. They keep + /// the production ratios that matter: the reference gate is at half the + /// size and block cost limits. + const SMALL: Limits = Limits { + source_bytes: 256, + line_nesting: 4, + container_markers: 16, + block_cost: 64 * 64, + table_cells: 32, + reference_lookups: 12, + html_bytes: 4 * 1024, + }; + + fn cost(text: &str) -> ParseCost { + ParseCost::of(text) + } + + fn renders_as_markdown(limits: &Limits, text: &str) -> bool { + render_gfm_bounded(text, limits).is_some() + } + + fn assert_plain(limits: &Limits, text: &str) { + let html = render_page_html_with(limits, text, text, true, None); + assert_eq!(html, plain_text_to_html(text), "{text:?}"); } /// Both rendering paths (page view and editor preview share @@ -393,7 +538,10 @@ mod tests { } /// Inputs the unpatched `markdown` crate panics on (see - /// `[patch.crates-io]` in the root Cargo.toml). + /// `[patch.crates-io]` in the root Cargo.toml): the regression inputs of + /// River's `malformed_markdown_renders_without_panicking`. Each goes + /// through both of the crate's code paths this module uses: HTML + /// rendering, and the syntax tree built for the exact reference count. #[test] fn malformed_markdown_renders_without_panicking() { let inputs = [ @@ -430,6 +578,11 @@ mod tests { ] { let rendered = std::panic::catch_unwind(|| render_every_way(&text)); assert!(rendered.is_ok(), "rendering {text:?} panicked"); + let walked = std::panic::catch_unwind(|| exact_reference_expansion(&text, 0)); + assert!( + matches!(walked, Ok(Some(_))), + "building the syntax tree of {text:?} failed" + ); } } } @@ -460,23 +613,34 @@ mod tests { let text: String = (0..len) .map(|_| PIECES[(next() % PIECES.len() as u64) as usize]) .collect(); - let rendered = std::panic::catch_unwind(|| render_every_way(&text)); + let rendered = std::panic::catch_unwind(|| { + render_every_way(&text); + exact_reference_expansion(&text, 0) + }); assert!(rendered.is_ok(), "rendering {text:?} panicked"); } } /// The AST helpers walk and drop a tree without recursion, so a deep tree /// cannot overflow the stack. Runs on a small stack to leave a margin - /// below wasm's 1 MiB. + /// below wasm's 1 MiB. The walk visits each node once: the count grows + /// with the nesting exactly. #[test] fn deep_markdown_trees_are_walked_without_recursion() { std::thread::Builder::new() .stack_size(256 * 1024) .spawn(|| { - let deep = format!("{}[a]\n\n[a]: b", ">".repeat(5_000)); - let tree = markdown::to_mdast(&deep, &markdown::ParseOptions::gfm()).unwrap(); - assert_eq!(reference_expansion(&tree, 0, 6, 64), 6 + 64); - drop_mdast(tree); + let walk = |depth: usize| { + let deep = format!("{}[a]\n\n[a]: b", ">".repeat(depth)); + exact_reference_expansion(&deep, 0).unwrap() + }; + // Root, `depth` quotes, a paragraph, a reference, its text, + // and the definition. + for depth in [1, 2, 4, 5_000] { + let expansion = walk(depth); + assert_eq!(expansion.html_bytes, 6 + 64); + assert_eq!(expansion.nodes, depth + 5, "{depth}"); + } }) .expect("spawn") .join() @@ -493,7 +657,7 @@ mod tests { loop { let line = format!("{}- [a]\n\n", " ".repeat(depth)); // Half the size limit, the most the exact reference count allows. - if text.len() + line.len() + 6_100 > MARKDOWN_MAX_SOURCE_BYTES / 2 { + if text.len() + line.len() + 6_100 > LIMITS.source_bytes / 2 { break; } text.push_str(&line); @@ -503,9 +667,9 @@ mod tests { // cheap reference bound fails and the tree is built and walked. text.push_str(&format!("[a]: b\n{}\n", "word ".repeat(1_200))); assert!(depth > 200, "{depth}"); - assert!(markdown_cost_is_bounded(&text)); + assert!(markdown_parse_cost(&text, &LIMITS).is_some()); let cheap = text.matches(']').count() * (longest_possible_definition(&text) * 6 + 64); - assert!(cheap > MARKDOWN_MAX_HTML_BYTES, "{cheap}"); + assert!(cheap > LIMITS.html_bytes, "{cheap}"); std::thread::Builder::new() .stack_size(256 * 1024) .spawn(move || { @@ -517,75 +681,158 @@ mod tests { .expect("rendering deep nesting panicked"); } - /// Deep block nesting on one line, a wide table, too much text, or too - /// much text without a blank line is shown as plain text, however the - /// line breaks are written. + /// The values the limits are set to. Changing one changes what readers + /// see, so it should be a deliberate edit here too. #[test] - fn costly_markdown_is_shown_as_plain_text() { - let deep_list = format!("{}x", "- ".repeat(MARKDOWN_MAX_LINE_NESTING + 1)); - let deep_quote = format!("{}x", "> 1. ".repeat(MARKDOWN_MAX_LINE_NESTING)); - // Pads every following line to 300 columns. - let wide_table = format!( - "{}\n{}\n{}", - "|a".repeat(300), - "|-".repeat(300), - "x\n".repeat(400) - ); - let quoted_wide_table = format!( - "> {}\n> {}\n{}", - "|a".repeat(300), - "| :-: ".repeat(300), - "x\n".repeat(400) + fn production_limits() { + let Limits { + source_bytes, + line_nesting, + container_markers, + block_cost, + table_cells, + reference_lookups, + html_bytes, + } = LIMITS; + assert_eq!(source_bytes, 131_072); + assert_eq!(line_nesting, 16); + assert_eq!(container_markers, 4_096); + assert_eq!(block_cost, 12_288 * 12_288); + assert_eq!(table_cells, 65_536); + assert_eq!(reference_lookups, 1_048_576); + assert_eq!(html_bytes, 4_194_304); + // And `render_page_html` applies them. + let deep = format!("{}x", "- ".repeat(line_nesting + 1)); + assert_eq!( + render_page_html(&deep, &deep, true, None), + plain_text_to_html(&deep) ); - // One paragraph past the block cost, written with each kind of line - // ending. - let long_paragraph = "word ".repeat(4 * 1024); - let crlf_paragraph = "word\r\n".repeat(4 * 1024); - let cr_paragraph = "word\r".repeat(4 * 1024); - // Many definitions and many closing brackets. - let lookups = format!("{}\n\n{}", "[a]: b\n\n".repeat(2_000), "x] ".repeat(1_000)); + let fine = format!("{}x", "- ".repeat(line_nesting)); + assert!(render_page_html(&fine, &fine, true, None).contains("
  • ")); + } + + /// The source size limit is exact. + #[test] + fn size_limit() { + let at = "a".repeat(SMALL.source_bytes); + let over = "a".repeat(SMALL.source_bytes + 1); + let limits = Limits { + block_cost: u64::MAX, + ..SMALL + }; + assert!(renders_as_markdown(&limits, &at)); + assert!(!renders_as_markdown(&limits, &over)); + assert_plain(&limits, &over); + } + + /// Containers opened on one line are counted however the line is + /// reached, and the per-line limit is exact. + #[test] + fn line_nesting_limit() { + let deep_list = format!("{}x", "- ".repeat(SMALL.line_nesting + 1)); + let deep_quote = format!("{}x", "> 1. ".repeat(SMALL.line_nesting / 2 + 1)); for costly in [ deep_list.clone(), format!("a\r{deep_list}"), format!("a\r\n{deep_quote}"), format!("[^a]: {deep_list}"), format!("\u{feff}{deep_list}"), - wide_table, - quoted_wide_table, - long_paragraph, - crlf_paragraph, - cr_paragraph, - lookups, - "a\n\n".repeat(MARKDOWN_MAX_SOURCE_BYTES / 3 + 1), ] { - let head = &costly[..costly.len().min(80)]; - assert!(!markdown_cost_is_bounded(&costly), "{head:?}"); - let html = render_page_html(&costly, &costly, true, None); - assert!(html.starts_with("

    "), "{head:?}"); - assert!(!html.contains("

  • ") && !html.contains("
    • "), "{head:?}"); - } - for fine in [ - format!("{}x", "- ".repeat(MARKDOWN_MAX_LINE_NESTING)), - "> quote\n- item\n 1. nested".to_string(), - "| a | b |\n| - | - |\n| 1 | 2 |".to_string(), - "-1 and 2.5 and -x".to_string(), - // A long code block of shell pipelines is not a wide table. - format!("```\n{}```", "a | b | c | d | e | f | g | h\n".repeat(200)), - ] { - assert!(markdown_cost_is_bounded(&fine), "{fine:?}"); + assert!( + cost(&costly).deepest_line > SMALL.line_nesting, + "{costly:?}" + ); + assert_plain(&SMALL, &costly); } + let at = format!("{}x", "- ".repeat(SMALL.line_nesting)); + assert_eq!(cost(&at).deepest_line, SMALL.line_nesting); + assert!(renders_as_markdown(&SMALL, &at)); + assert_eq!(line_container_depth(" > - 1) * x"), 4); assert_eq!(line_container_depth(">>> x"), 3); assert_eq!(line_container_depth("-x"), 0); assert_eq!(line_container_depth("2024. was"), 1); assert_eq!(line_container_depth("[^a]: > - x"), 3); assert_eq!(line_container_depth("[^a] x"), 0); - assert_eq!(delimiter_row_columns("| --- | :-: |"), Some(2)); - assert_eq!(delimiter_row_columns("> -|-"), Some(2)); - assert_eq!(delimiter_row_columns("| a |"), None); - assert_eq!(delimiter_row_columns("| | |"), None); - assert_eq!(delimiter_row_columns("---"), Some(1)); - assert_eq!(delimiter_row_columns("|-|-|-|-|"), Some(4)); + assert_eq!(line_container_depth("-1 and 2.5 and -x"), 0); + } + + /// The whole-text container count is the sum of each line's, so it grows + /// linearly with the text, and its limit is exact. + #[test] + fn container_marker_limit() { + for lines in [1, 2, 4, 8] { + assert_eq!(cost(&"- a\n".repeat(lines)).container_markers, lines); + assert_eq!(cost(&"> - a\n".repeat(lines)).container_markers, 2 * lines); + assert_eq!(cost(&"- a\n\n".repeat(lines)).container_markers, lines); + } + let limits = Limits { + block_cost: u64::MAX, + ..SMALL + }; + let at = "- a\n".repeat(limits.container_markers); + let over = "- a\n".repeat(limits.container_markers + 1); + assert!(renders_as_markdown(&limits, &at)); + assert!(!renders_as_markdown(&limits, &over)); + assert_plain(&limits, &over); + // Spread over lines that each stay well within the per-line limit. + let quoted = "> > a\n\n".repeat(limits.container_markers / 2 + 1); + assert!(cost("ed).deepest_line <= limits.line_nesting); + assert!(!renders_as_markdown(&limits, "ed)); + } + + /// A byte order mark is skipped only at the very start, as the parser + /// does. Anywhere else it is text, and counts as text. + #[test] + fn byte_order_mark_is_skipped_only_at_the_start() { + // A leading one does not hide the markers after it. + let deep = format!("\u{feff}{}x", "- ".repeat(SMALL.line_nesting + 1)); + assert_eq!(cost(&deep).deepest_line, SMALL.line_nesting + 1); + assert_plain(&SMALL, &deep); + assert!(markdown::to_html("\u{feff}- x").contains("
  • ")); + + // A second one is text: no list opens, for the parser or the count. + assert_eq!(cost("\u{feff}\u{feff}- x").container_markers, 0); + assert!(!markdown::to_html("\u{feff}\u{feff}- x").contains("
  • ")); + + // Marks inside a paragraph count toward its length (3 bytes each). + for marks in [1, 2, 4, 8] { + let text = format!("a{}", "\u{feff}".repeat(marks)); + let run = (1 + 3 * marks + 1) as u64; + assert_eq!(cost(&text).block, run * run, "{marks}"); + } + let marks = "\u{feff}".repeat(30); + assert!(cost(&marks).block > SMALL.block_cost); + assert_plain(&SMALL, &marks); + } + + /// Block cost is the sum of each run's length squared: quadratic in a + /// run, linear in the number of runs. The limit is exact. + #[test] + fn block_cost_limit() { + for lines in [1, 2, 4] { + let run = 3 * lines as u64; + assert_eq!(cost(&"ab\n".repeat(lines)).block, run * run); + assert_eq!(cost(&"ab\r\n".repeat(lines)).block, run * run); + assert_eq!(cost(&"ab\r".repeat(lines)).block, run * run); + assert_eq!(cost(&"ab\n\n".repeat(lines)).block, 9 * lines as u64); + } + // 63 bytes and a line ending: exactly the 64 x 64 limit. + let at = "a".repeat(63); + let over = "a".repeat(64); + assert_eq!(cost(&at).block, SMALL.block_cost); + assert!(renders_as_markdown(&SMALL, &at)); + assert!(!renders_as_markdown(&SMALL, &over)); + // Each line ending, however written, keeps the run going. + for ending in ["\n", "\r\n", "\r"] { + let paragraph = format!("word{ending}").repeat(16); + assert!(cost(¶graph).block > SMALL.block_cost, "{ending:?}"); + assert_plain(&SMALL, ¶graph); + } + // The same text split by blank lines is fine. + assert!(renders_as_markdown(&SMALL, &"word\n\n".repeat(16))); + assert!(!is_blank_line("\u{a0}")); + assert!(is_blank_line(" \t")); } /// Lines split the way markdown splits them: a `\r\n` is one line ending, @@ -595,77 +842,253 @@ mod tests { let lines: Vec<&str> = markdown_lines("a\r\nb\rc\n\nd").collect(); assert_eq!(lines, ["a", "b", "c", "", "d"]); assert_eq!(markdown_lines("").collect::>(), [""]); - assert!(!is_blank_line("\u{a0}")); - assert!(is_blank_line(" \t")); } - /// A long document of ordinary markdown, near the size limit, renders as - /// markdown. + /// Table cells are the widest delimiter row so far times the rows after + /// it in the run, and the limit is exact. + #[test] + fn table_cell_limit() { + // A 4-column table with `rows` body rows. + let table = |rows: usize| format!("|a|b|c|d|\n|-|-|-|-|\n{}", "x\n".repeat(rows)); + for rows in [0, 1, 2, 4] { + // The delimiter row itself counts as one row. + assert_eq!(cost(&table(rows)).table_cells, 4 * (rows + 1)); + } + let limits = Limits { + block_cost: u64::MAX, + ..SMALL + }; + assert!(renders_as_markdown(&limits, &table(7))); + assert!(!renders_as_markdown(&limits, &table(8))); + assert_plain(&limits, &table(8)); + let quoted = format!("> |a|b|\n> | :-: | :-: |\n{}", "x\n".repeat(16)); + assert!(!renders_as_markdown(&limits, "ed)); + // A code block of shell pipelines is not a table. + let pipes = format!("```\n{}```", "a | b | c\n".repeat(8)); + assert_eq!(cost(&pipes).table_cells, 0); + + assert_eq!(delimiter_row_columns("| --- | :-: |"), Some(2)); + assert_eq!(delimiter_row_columns("> -|-"), Some(2)); + assert_eq!(delimiter_row_columns("| a |"), None); + assert_eq!(delimiter_row_columns("| | |"), None); + assert_eq!(delimiter_row_columns("---"), Some(1)); + assert_eq!(delimiter_row_columns("|-|-|-|-|"), Some(4)); + } + + /// `]:` count times `]` count, with an exact limit. + #[test] + fn reference_lookup_limit() { + // `]:` itself contains a `]`. + assert_eq!(cost("]:]:]:]").reference_lookups, 3 * 4); + assert_eq!(cost("]:]:]:]]").reference_lookups, 3 * 5); + assert_eq!(cost("]]]]").reference_lookups, 0); + assert!(renders_as_markdown(&SMALL, "]:]:]:]")); + assert!(!renders_as_markdown(&SMALL, "]:]:]:]]")); + assert_plain(&SMALL, "]:]:]:]]"); + } + + /// The exact reference count: each reference costs its definition's + /// destination and title, six times over, plus 64 bytes. Only definitions + /// that something refers to count, and the first definition of a label + /// wins. + #[test] + fn exact_reference_count() { + let count = |text: &str| exact_reference_expansion(text, 0).unwrap().html_bytes; + // Destination 5 bytes, title 3. + let one = (5 + 3) * 6 + 64; + for refs in [1, 2, 4] { + let text = format!("[a]: bbbbb 'ttt'\n\n{}", "[a] ".repeat(refs)); + assert_eq!(count(&text), refs * one, "{refs}"); + } + assert_eq!(count("[a]: bbbbb 'ttt'\n\n[z]"), 0); + assert_eq!(count("[a]: b\n\n[a]: ccccc\n\n[a]"), 6 + 64); + assert_eq!(count("[a]: bb\n\n![x][a]"), 2 * 6 + 64); + } + + /// The exact count is used only when the cheap bound fails, and only for + /// text within half the size and block cost limits. #[test] - fn long_ordinary_document_renders_as_markdown() { + fn exact_reference_count_is_gated_at_half_the_limits() { + // A titled definition followed by more text in its run: the cheap + // bound assumes the title may run to the end of the run. + let text_of_len = |len: usize| { + let head = "[a]: b 't'\nx"; + format!("{head}{}\n\n[a]", "y".repeat(len - head.len() - 5)) + }; + let limits = Limits { + html_bytes: 64, + block_cost: u64::MAX, + ..SMALL + }; + let half = limits.source_bytes / 2; + let at = text_of_len(half); + let over = text_of_len(half + 1); + assert_eq!((at.len(), over.len()), (half, half + 1)); + for text in [&at, &over] { + let cheap = text.matches(']').count() * (longest_possible_definition(text) * 6 + 64); + assert!(cheap > limits.html_bytes); + // Destination and title are one byte each. + assert_eq!(count_html(text), 2 * 6 + 64); + } + let cost_at = markdown_parse_cost(&at, &limits).unwrap(); + let cost_over = markdown_parse_cost(&over, &limits).unwrap(); + // The exact count (76 bytes) is over 64, so lift that for this check. + let roomy = Limits { + html_bytes: 80, + ..limits + }; + assert!(references_are_bounded(&at, &cost_at, &roomy)); + assert!(!references_are_bounded(&over, &cost_over, &roomy)); + assert!(!references_are_bounded(&at, &cost_at, &limits)); + + // The same at half the block cost. + let block_limits = Limits { + block_cost: 2 * cost_at.block, + ..roomy + }; + assert!(references_are_bounded(&at, &cost_at, &block_limits)); + let block_limits = Limits { + block_cost: 2 * cost_at.block - 1, + ..roomy + }; + assert!(!references_are_bounded(&at, &cost_at, &block_limits)); + + fn count_html(text: &str) -> usize { + exact_reference_expansion(text, 0).unwrap().html_bytes + } + } + + /// The cheap reference bound lets many short untitled definitions through + /// without a second parse, and the exact count catches a long title used + /// many times. + #[test] + fn reference_expansion_falls_back_to_plain_text() { + let limits = Limits { + block_cost: u64::MAX, + ..SMALL + }; + // 4 references to a 30-byte title: 4 x 244 bytes. + let text = format!("[a]: b '{}'\n\n{}", "\"".repeat(29), "[a]\n\n".repeat(4)); + assert!(text.len() <= limits.source_bytes / 2); + let expanded = exact_reference_expansion(&text, 0).unwrap().html_bytes; + assert_eq!(expanded, 4 * (30 * 6 + 64)); + let tight = Limits { + html_bytes: expanded - 1, + ..limits + }; + assert!(!renders_as_markdown(&tight, &text)); + assert_plain(&tight, &text); + + let untitled = "See [r1] and [r2].\n\n[r1]: /a\n[r2]: /b\n"; + assert!(longest_possible_definition(untitled) < 16); + let html = render_page_html_with(&limits, untitled, untitled, true, None); + assert!(html.contains("href=\"/b\""), "{html}"); + } + + /// The HTML is capped twice: as the parser writes it, and after heading + /// ids and link attributes are added. Each cap is exact. + #[test] + fn html_output_caps() { + let text = "# Head\n\n[x](https://e.example)"; + let raw = markdown::to_html_with_options(text, &markdown::Options::gfm()).unwrap(); + let finished = render_page_html_with(&SMALL, text, text, true, None); + assert!(finished.len() > raw.len(), "{finished}"); + assert!(finished.contains("target=\"_blank\"") && finished.contains("id=\"head\"")); + + let with_cap = |html_bytes| Limits { + html_bytes, + ..SMALL + }; + let render = |cap| render_page_html_with(&with_cap(cap), text, text, true, None); + // Parser output over the cap. + assert_eq!(render(raw.len() - 1), plain_text_to_html(text)); + assert!(render_gfm_bounded(text, &with_cap(raw.len())).is_some()); + // Parser output within the cap, finished HTML over it. + assert_eq!(render(raw.len()), plain_text_to_html(text)); + assert_eq!(render(finished.len() - 1), plain_text_to_html(text)); + assert_eq!(render(finished.len()), finished); + } + + /// A long document of ordinary markdown renders as markdown. Its cost + /// grows linearly with its length, so at the production size limit it + /// stays within every other production limit. + #[test] + fn ordinary_markdown_renders_and_scales_linearly() { let section = "## A heading\n\n\ A paragraph with **bold**, _emphasis_, `code`, a [link](https://example.com/a) \ and a [reference link][r]. It goes on for a while, the way paragraphs in \ long documents do, with more words and another [link](#a-heading).\n\n\ - - item one\n- item two\n 1. nested\n\n\ + - item one\n- item two\n 1. nested\n- [ ] a task\n\n\ | a | b | c |\n|---|:-:|--:|\n| 1 | 2 | 3 |\n| 4 | 5 | 6 |\n\n\ ```\nfn main() { println!(\"hi\"); }\n```\n\n\ > A quote.\n\n"; - let mut text = String::new(); - while text.len() + section.len() < MARKDOWN_MAX_SOURCE_BYTES - 64 { - text.push_str(section); - } - text.push_str("[r]: https://example.com/r \"A title\"\n"); - assert!(markdown_cost_is_bounded(&text)); + let definition = "[r]: https://example.com/r \"A title\"\n"; + let doc = |sections: usize| format!("{}{definition}", section.repeat(sections)); + + let one = cost(&doc(1)); + let two = cost(&doc(2)); + let four = cost(&doc(4)); + assert_eq!(four.container_markers, 4 * one.container_markers); + assert_eq!(four.table_cells, 4 * one.table_cells); + // Runs do not grow, so block cost is linear (the last run differs). + assert_eq!(four.block - two.block, 2 * (two.block - one.block)); + + let sections_at_limit = LIMITS.source_bytes / section.len(); + let per_section = |field: fn(&ParseCost) -> u64| field(&two) - field(&one); + let at_limit = |field: fn(&ParseCost) -> u64| { + field(&one) + per_section(field) * sections_at_limit as u64 + }; + assert!(at_limit(|c| c.block) <= LIMITS.block_cost / 2); + assert!(at_limit(|c| c.container_markers as u64) <= LIMITS.container_markers as u64); + assert!(at_limit(|c| c.table_cells as u64) <= LIMITS.table_cells as u64); + assert!(at_limit(|c| c.deepest_line as u64) <= LIMITS.line_nesting as u64); + + let text = doc(4); let html = render_page_html(&text, &text, true, None); assert!(html.contains("

    "), "{}", &html[..300]); assert!(html.contains("") && html.contains("")); assert!(html.contains("href=\"https://example.com/r\" title=\"A title\"")); } - /// A definition with a long title used many times would expand to far - /// more HTML than the limit, so it is caught before rendering. + /// The page view checks the size before resolving page links, and shows + /// the content as written if resolving would make it too long. #[test] - fn references_that_expand_too_far_are_shown_as_plain_text() { - let text = format!( - "[a]: b '{}'\n\n{}", - "\"".repeat(10_000), - "[a]\n\n".repeat(10_000) + fn page_view_checks_size_before_resolving_links() { + let over = "a".repeat(SMALL.source_bytes + 1); + let html = render_page_view_html_with( + &SMALL, + &over, + |_, _| panic!("resolved links in text over the size limit"), + true, + None, ); - assert!(markdown_cost_is_bounded(&text)); - assert!(!references_bounded(&text)); - let html = render_page_html(&text, &text, true, None); - assert!(html.starts_with("

    [a]: b"), "{}", &html[..100]); - assert!(html.len() < 8 * text.len()); - - // Many reference links to short titled definitions are fine, although - // the cheap bound alone would reject them (a title may run to the end - // of the run, and the definitions are one run). - let mut many = String::new(); - for i in 0..150 { - many.push_str(&format!("See [this][r{i}] and [that][r{i}].\n\n")); - } - for i in 0..150 { - many.push_str(&format!("[r{i}]: https://example.com/{i} \"Title {i}\"\n")); - } - assert!(markdown_cost_is_bounded(&many)); - let cheap = many.matches(']').count() * (longest_possible_definition(&many) * 6 + 64); - assert!(cheap > MARKDOWN_MAX_HTML_BYTES, "{cheap}"); - assert!(references_bounded(&many)); - let html = render_page_html(&many, &many, true, None); - assert!(html.contains("href=\"https://example.com/149\" title=\"Title 149\"")); - - // Untitled definitions have a tight cheap bound, so a long document of - // them needs no second parse. - let mut untitled = String::new(); - for i in 0..300 { - untitled.push_str(&format!("See [this][r{i}].\n\n")); - } - for i in 0..300 { - untitled.push_str(&format!("[r{i}]: https://example.com/{i}\n")); - } - assert!(longest_possible_definition(&untitled) < 100); - assert!(render_page_html(&untitled, &untitled, true, None).contains("/299\"")); + assert_eq!(html, plain_text_to_html(&over)); + + let content = "[[Page]] "; + let mut max_seen = None; + let html = render_page_view_html_with( + &SMALL, + content, + |text, max| { + assert_eq!(text, content); + max_seen = Some(max); + None + }, + true, + None, + ); + assert_eq!(max_seen, Some(SMALL.source_bytes)); + assert_eq!(html, "

    [[Page]] <b>

    "); + + // Resolved text goes through the limits, and the plain text shown is + // the content as written. + let deep = format!("{}x", "- ".repeat(SMALL.line_nesting + 1)); + let html = + render_page_view_html_with(&SMALL, "[[P]]", |_, _| Some(deep.clone()), true, None); + assert_eq!(html, "

    [[P]]

    "); + let html = + render_page_view_html_with(&SMALL, "[[P]]", |_, _| Some("[P](#p)".into()), true, None); + assert!(html.contains("P"), "{html}"); } /// Text not rendered as markdown is escaped, keeps its line breaks, and @@ -677,4 +1100,103 @@ mod tests { assert_eq!(html, "

    <b>[[Page]]</b>
    \nsecond

    "); assert!(!render_page_html(&costly, &costly, true, None).contains("")); } + + /// Every use of the `markdown` crate goes through this module, and both + /// the page view and the editor preview render through it. A component + /// that called the crate directly, or stopped calling the bounded entry + /// points, fails here. + #[test] + fn page_view_and_editor_render_through_this_module() { + /// Source with `//` comments removed, so a comment mentioning a call + /// does not count as one. + fn code(source: &str) -> String { + source + .lines() + .map(|line| line.split("//").next().unwrap_or("")) + .collect::>() + .join("\n") + } + fn uses_crate(code: &str, krate: &str) -> bool { + code.match_indices(&format!("{krate}::")).any(|(i, _)| { + !code[..i] + .chars() + .next_back() + .is_some_and(|c| c.is_alphanumeric() || c == '_') + }) + } + /// The body of `fn name` in `code`: from its signature to the first + /// line that closes an item at the left margin. + fn body<'a>(code: &'a str, name: &str) -> &'a str { + let start = code + .find(&format!("fn {name}(")) + .unwrap_or_else(|| panic!("no fn {name}")); + let end = code[start..].find("\n}").expect("item end"); + &code[start..start + end] + } + let module = concat!("mark", "down_render::"); + + let editor = code(include_str!("editor.rs")); + let editor_fn = body(&editor, "Editor"); + assert!( + editor_fn.contains(&format!("{module}render_page_html(")), + "the editor preview no longer renders through render_page_html" + ); + + let page_view = code(include_str!("page_view.rs")); + assert!( + body(&page_view, "PageView").contains("render_markdown("), + "the page view no longer renders through render_markdown" + ); + assert!( + body(&page_view, "render_markdown").contains("page_content_html("), + "render_markdown no longer renders through page_content_html" + ); + assert!( + body(&page_view, "page_content_html") + .contains(&format!("{module}render_page_view_html(")), + "page_content_html no longer renders through render_page_view_html" + ); + + let src = concat!(env!("CARGO_MANIFEST_DIR"), "/src"); + let mut dirs = vec![std::path::PathBuf::from(src)]; + let mut scanned = 0; + while let Some(dir) = dirs.pop() { + for entry in std::fs::read_dir(&dir).expect("read src") { + let path = entry.expect("entry").path(); + if path.is_dir() { + dirs.push(path); + } else if path.extension().is_some_and(|e| e == "rs") + && !path.ends_with("components/markdown_render.rs") + { + let source = std::fs::read_to_string(&path).expect("read"); + assert!( + !uses_crate(&code(&source), "markdown"), + "{} uses the markdown crate directly", + path.display() + ); + scanned += 1; + } + } + } + assert!(scanned > 10, "scanned only {scanned} files"); + assert!(uses_crate("x = markdown::to_html(t)", "markdown")); + assert!(!uses_crate("super::render_markdown::x()", "markdown")); + } + + /// The patched `markdown` build is the one compiled in. The parser fixes + /// are tested above; the last commit (linear-time parsing) changes no + /// output, so this pin is what keeps it. + #[test] + fn patched_markdown_build_is_pinned() { + let lock = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/../Cargo.lock")) + .expect("read Cargo.lock"); + let rev = "c0646ed72010008a3ea92bf52be43082a09bdd12"; + let source = format!("git+https://github.com/freenet/markdown-rs?rev={rev}#{rev}"); + let packages: Vec<&str> = lock + .split("[[package]]") + .filter(|p| p.contains("\nname = \"markdown\"\n")) + .collect(); + assert_eq!(packages.len(), 1, "one markdown package in Cargo.lock"); + assert!(packages[0].contains(&source), "{}", packages[0]); + } } diff --git a/ui/src/components/page_view.rs b/ui/src/components/page_view.rs index c27d686..8295449 100644 --- a/ui/src/components/page_view.rs +++ b/ui/src/components/page_view.rs @@ -3,6 +3,7 @@ use dioxus::prelude::*; use crate::state; use crate::state::SiteRole; use delta_core::PageId; +use std::collections::{BTreeMap, HashMap}; #[component] pub fn PageView() -> Element { @@ -202,15 +203,34 @@ pub fn PageView() -> Element { /// Content too costly to render as markdown is shown as plain text (see /// `markdown_render`). fn render_markdown(content: &str) -> String { - let resolved = resolve_page_links(content); - super::markdown_render::render_page_html( - &resolved, + let prefix = state::CURRENT_SITE.read().clone().unwrap_or_default(); + let sites = state::SITES.read(); + let pages = sites.get(&prefix).map(|s| &s.state.pages); + page_content_html( content, + &prefix, + pages, behind_gateway(), own_contract_id().as_deref(), ) } +/// `render_markdown` for the site `prefix` with `pages`. +fn page_content_html( + content: &str, + prefix: &str, + pages: Option<&BTreeMap>, + rewrite_freenet_hrefs: bool, + own_contract_id: Option<&str>, +) -> String { + super::markdown_render::render_page_view_html( + content, + |text, max_len| resolve_page_links(text, prefix, pages, max_len, &mut 0), + rewrite_freenet_hrefs, + own_contract_id, + ) +} + /// True when Delta is currently being served from a path under /// `/v1/contract/web/`, which is what gateway hosting looks like to the /// browser. Returning false suppresses the host-stripping href rewrite @@ -670,87 +690,134 @@ fn slugify_heading(text: &str) -> String { out } -/// Replace page links with hash-routed markdown links. +/// Replace page links with hash-routed markdown links, or return `None` if +/// the result would be longer than `max_len` or the work would exceed +/// `RESOLVE_WORK_PER_BYTE * max_len`. /// /// Supported syntax: /// [[id|Display Text]] - link by page ID (canonical, stored format) /// [[Page Title]] - link by page title (user-friendly) /// [[Page Title|Label]] - link by title with custom display text -fn resolve_page_links(content: &str) -> String { - let prefix = state::CURRENT_SITE.read().clone().unwrap_or_default(); - let sites = state::SITES.read(); - let pages = sites.get(&prefix).map(|s| &s.state.pages); - - let mut result = String::with_capacity(content.len()); +/// +/// `work` counts the bytes searched for `[[` and `]]`, plus the bytes of +/// each page title a link copies. Each byte of `content` is searched at most +/// once, and scanning stops at the last `]]`. +fn resolve_page_links( + content: &str, + prefix: &str, + pages: Option<&BTreeMap>, + max_len: usize, + work: &mut usize, +) -> Option { + let work_budget = max_len.saturating_mul(RESOLVE_WORK_PER_BYTE); + let mut titles = TitleIndex::new(pages); + let mut result = String::with_capacity(content.len().min(max_len)); let mut rest = content; - while let Some(start) = rest.find("[[") { - result.push_str(&rest[..start]); + loop { + let Some(start) = rest.find("[[") else { + *work += rest.len(); + break; + }; + *work += start + 2; let after_open = &rest[start + 2..]; + let Some(end) = after_open.find("]]") else { + // No `]]` remains, so no later `[[` can open a link either. + *work += after_open.len(); + break; + }; + *work += end + 2; + result.push_str(&rest[..start]); - if let Some(end) = after_open.find("]]") { - let link_content = &after_open[..end]; - let resolved = if let Some((first, display)) = link_content.split_once('|') { - // [[first|display]] - first could be ID or title - // The display text is always used as the rendered link text - if let Ok(id) = first.trim().parse::() { - // [[id|Display Text]] - canonical format - let slug = pages - .and_then(|p| p.get(&id)) - .map(|p| p.title.clone()) - .unwrap_or_else(|| display.to_string()); - let hash = state::build_hash_route(&prefix, Some(id), Some(&slug)); - Some(format!("[{display}]({hash})")) - } else { - // [[Title|Label]] - look up by title, show label - find_page_by_title(pages, first.trim()).map(|(id, _)| { - let hash = state::build_hash_route(&prefix, Some(id), Some(first.trim())); - format!("[{display}]({hash})") - }) - } + let link_content = &after_open[..end]; + let resolved = if let Some((first, display)) = link_content.split_once('|') { + // [[first|display]] - first could be ID or title + // The display text is always used as the rendered link text + if let Ok(id) = first.trim().parse::() { + // [[id|Display Text]] - canonical format + let slug = match pages.and_then(|p| p.get(&id)) { + Some(page) => { + *work += page.title.len(); + page.title.as_str() + } + None => display, + }; + let hash = state::build_hash_route(prefix, Some(id), Some(slug)); + Some(format!("[{display}]({hash})")) } else { - // [[id]] or [[Page Title]] - no custom display text - let trimmed = link_content.trim(); - if let Ok(id) = trimmed.parse::() { - // [[id]] - render as current page title (auto-updates on rename) - pages.and_then(|p| p.get(&id)).map(|p| { - let hash = state::build_hash_route(&prefix, Some(id), Some(&p.title)); - format!("[{}]({hash})", p.title) - }) - } else { - // [[Page Title]] - look up by title - find_page_by_title(pages, trimmed).map(|(id, title)| { - let hash = state::build_hash_route(&prefix, Some(id), Some(&title)); - format!("[{title}]({hash})") - }) - } + // [[Title|Label]] - look up by title, show label + titles.find(first.trim()).map(|(id, _)| { + let hash = state::build_hash_route(prefix, Some(id), Some(first.trim())); + format!("[{display}]({hash})") + }) + } + } else { + // [[id]] or [[Page Title]] - no custom display text + let trimmed = link_content.trim(); + let page = if let Ok(id) = trimmed.parse::() { + // [[id]] - render as current page title (auto-updates on rename) + pages + .and_then(|p| p.get(&id)) + .map(|p| (id, p.title.as_str())) + } else { + // [[Page Title]] - look up by title + titles.find(trimmed) }; + page.map(|(id, title)| { + *work += title.len(); + let hash = state::build_hash_route(prefix, Some(id), Some(title)); + format!("[{title}]({hash})") + }) + }; - result.push_str(&resolved.unwrap_or_else(|| { - // Broken link - render as styled warning text - format!("[[{link_content}]]") - })); - rest = &after_open[end + 2..]; - } else { - result.push_str("[["); - rest = after_open; + result.push_str(&resolved.unwrap_or_else(|| { + // Broken link - render as styled warning text + format!("[[{link_content}]]") + })); + rest = &after_open[end + 2..]; + if result.len() > max_len || *work > work_budget { + return None; } } result.push_str(rest); - result + (result.len() <= max_len).then_some(result) +} + +/// Most work `resolve_page_links` does per byte of its output limit: one for +/// searching the content, the rest for the page titles links copy. +const RESOLVE_WORK_PER_BYTE: usize = 4; + +/// Pages by title, case-insensitively, for `[[Page Title]]` links. Built on +/// first use, so a page with no title links never builds it, and a page with +/// many builds it once. +struct TitleIndex<'a> { + pages: Option<&'a BTreeMap>, + by_title: Option>, } -/// Find a page by title (case-insensitive). -fn find_page_by_title( - pages: Option<&std::collections::BTreeMap>, - title: &str, -) -> Option<(PageId, String)> { - let pages = pages?; - let lower = title.to_lowercase(); - pages - .iter() - .find(|(_, p)| p.title.to_lowercase() == lower) - .map(|(&id, p)| (id, p.title.clone())) +impl<'a> TitleIndex<'a> { + fn new(pages: Option<&'a BTreeMap>) -> Self { + Self { + pages, + by_title: None, + } + } + + /// The page titled `title`, ignoring case. If several are, the one with + /// the lowest id. + fn find(&mut self, title: &str) -> Option<(PageId, &'a str)> { + let pages = self.pages?; + let by_title = self.by_title.get_or_insert_with(|| { + let mut by_title = HashMap::new(); + for (&id, page) in pages { + by_title + .entry(page.title.to_lowercase()) + .or_insert((id, page.title.as_str())); + } + by_title + }); + by_title.get(&title.to_lowercase()).copied() + } } /// Copy the full URL for a specific page to clipboard. @@ -1327,4 +1394,130 @@ mod tests { // Internal hash anchor untouched. assert!(result.contains("href=\"#section\">jump")); } + + mod page_links { + use super::super::{page_content_html, resolve_page_links, RESOLVE_WORK_PER_BYTE}; + use delta_core::{Page, PageId}; + use std::collections::BTreeMap; + + const PREFIX: &str = "AbCdEfGhJk"; + + fn pages(titles: &[(PageId, &str)]) -> BTreeMap { + let key = ed25519_dalek::SigningKey::from_bytes(&[7u8; 32]); + titles + .iter() + .map(|&(id, title)| (id, Page::new(id, title.into(), String::new(), 1, &key))) + .collect() + } + + fn resolve(content: &str, pages: &BTreeMap, max: usize) -> Option { + resolve_page_links(content, PREFIX, Some(pages), max, &mut 0) + } + + /// Work done resolving `content`, with no output limit. + fn work(content: &str, pages: &BTreeMap) -> usize { + let mut work = 0; + resolve_page_links(content, PREFIX, Some(pages), usize::MAX / 8, &mut work); + work + } + + #[test] + fn links_resolve_by_id_and_title() { + let pages = pages(&[(1, "Home"), (2, "About Us"), (3, "home")]); + let resolved = resolve( + "[[1]] [[2|who]] [[ABOUT US]] [[about us|team]] [[home]] [[9]] [[x|y]] [[", + &pages, + 1024, + ) + .unwrap(); + assert_eq!( + resolved, + "[Home](#AbCdEfGhJk/1/home) [who](#AbCdEfGhJk/2/about-us) \ + [About Us](#AbCdEfGhJk/2/about-us) [team](#AbCdEfGhJk/2/about-us) \ + [Home](#AbCdEfGhJk/1/home) \ + [[9]] \ + [[x|y]] [[" + ); + } + + /// Searching visits each byte once, however many `[[` have no `]]` + /// after them: the work equals the length. + #[test] + fn scanning_is_linear() { + let pages = pages(&[(1, "A")]); + for n in [1, 2, 4, 8, 16] { + let unclosed = "[[".repeat(n); + assert_eq!(work(&unclosed, &pages), unclosed.len(), "{n}"); + assert_eq!(resolve(&unclosed, &pages, 1024).unwrap(), unclosed); + + let nested = format!("{}x]]", "[[".repeat(n)); + assert_eq!(work(&nested, &pages), nested.len(), "{n}"); + + // Links, then unclosed openers: the work is the length plus + // each link's title. + let mixed = format!("{}{}", "[[1]] ".repeat(n), "[[ ".repeat(n)); + assert_eq!(work(&mixed, &pages), mixed.len() + n, "{n}"); + } + } + + /// The output limit is exact. + #[test] + fn output_is_capped() { + let pages = pages(&[(1, "A page with a long title")]); + let content = "[[1]] [[missing]] [[1]]"; + let full = resolve(content, &pages, usize::MAX / 8).unwrap(); + assert!(full.len() > content.len()); + assert_eq!(resolve(content, &pages, full.len()), Some(full.clone())); + assert_eq!(resolve(content, &pages, full.len() - 1), None); + // Text without links is checked too. + assert_eq!(resolve("abc", &pages, 2), None); + } + + /// Copying page titles counts as work, so many links to a page with a + /// long title that adds little output still stop. + #[test] + fn title_work_is_capped() { + let title = "!".repeat(100); + let pages = pages(&[(1, title.as_str())]); + // Each link is 7 bytes searched and 100 of title, and its slug + // is empty, so its output is short. + let max = 128; + let budget = max * RESOLVE_WORK_PER_BYTE; + for links in 1..=8 { + let content = "[[1|d]]".repeat(links); + let expected_work = links * (7 + 100); + assert_eq!(work(&content, &pages), expected_work); + let resolved = resolve(&content, &pages, max); + assert_eq!(resolved.is_some(), expected_work <= budget, "{links}"); + } + } + + /// The page view renders through the bounded renderer: costly content + /// is plain text, shown as written, and links resolve otherwise. + #[test] + fn page_view_renders_through_the_bounded_renderer() { + let pages = pages(&[(1, "Home")]); + let render = + |content: &str| page_content_html(content, PREFIX, Some(&pages), true, None); + + let html = render("See [[1]].\n\n- item"); + assert!( + html.contains("Home"), + "{html}" + ); + assert!(html.contains("
  • item
    • "), "{html}"); + + let deep = format!("[[1]]\n{}x", "- ".repeat(17)); + assert_eq!( + render(&deep), + format!("

    [[1]]
    \n{}x

    ", "- ".repeat(17)) + ); + + // Malformed input from the parser fixes, reached through a link. + for input in ["[[1]] [a](b \"x\ny\")", "[[1]]\n1.