diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index ddd3c13d..77ed2c04 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -194,7 +194,9 @@ fn search_query(s: &mut Session, p: &Params) -> Result { Some(l) => l.trim().parse::().map_err(|_| EngineError::BadParam("limit must be a number".into()))?.min(500), None => 40, }; - Ok(Output::Hits(search::search(s.root(), req(p, "q")?, limit))) + // Every open notebook, the ones opened from another folder too: the window calls this + // Search all pages (issue #127). + Ok(Output::Hits(search::search_in(s.root(), s.opened_elsewhere(), req(p, "q")?, limit))) } fn notebook_check(s: &mut Session, p: &Params) -> Result { @@ -672,6 +674,37 @@ mod tests { assert_eq!(std::fs::read_dir(&bare).into_iter().flatten().flatten().filter(|e| e.path().is_dir()).count(), 0); } + /// Issue #127: Search all pages finds pages in a notebook opened from another folder, once + /// each, with the limit applied to every notebook together. + #[test] + fn search_finds_pages_in_a_notebook_opened_from_elsewhere() { + let t = TempDir::new("engine-search-elsewhere"); + let settings = Settings::at(t.path().join("settings").join("s.toml")); + let mut s = Session::new(&t.path().join("notebooks"), settings.clone()).expect("session"); + let here = path_of(s.run("notebook.create", &p(&[("name", "Here")])).expect("notebook")); + let here_page = here.join("new-section-1").join("untitled-page"); + s.run("page.save", &p(&[("path", &here_page.to_string_lossy()), ("body", "

shared_token here

")])).expect("save"); + + let away = t.path().join("away"); + std::fs::create_dir_all(&away).expect("dir"); + let mut other = Session::new(&away, settings.clone()).expect("other session"); + let made = path_of(other.run("notebook.create", &p(&[("name", "Elsewhere")])).expect("notebook")); + let away_page = made.join("new-section-1").join("untitled-page"); + other + .run("page.save", &p(&[("path", &away_page.to_string_lossy()), ("body", "

external_unique_token and shared_token

")])) + .expect("save"); + s.run("notebook.open", &p(&[("path", &made.to_string_lossy())])).expect("open"); + + let hits = |s: &mut Session, q: &str, limit: &str| match s.run("search.query", &p(&[("q", q), ("limit", limit)])).expect("search") { + Output::Hits(h) => h, + other => panic!("{other:?}"), + }; + let found = hits(&mut s, "external_unique_token", "10"); + assert_eq!(found.len(), 1, "the page in the notebook opened from elsewhere is found: {found:?}"); + assert_eq!(hits(&mut s, "shared_token", "10").len(), 2, "a page in each notebook, each once"); + assert_eq!(hits(&mut s, "shared_token", "1").len(), 1, "the limit is for all the notebooks together"); + } + #[test] fn a_notebook_from_elsewhere_opens_beside_the_others_and_stays_open() { let t = TempDir::new("engine-open-elsewhere"); diff --git a/crates/formats/src/csv.rs b/crates/formats/src/csv.rs index 71db9543..6b68bb11 100644 --- a/crates/formats/src/csv.rs +++ b/crates/formats/src/csv.rs @@ -3,6 +3,9 @@ //! The marks inside a cell are dropped, because a spreadsheet cell holds a value, not a //! marked-up line. Quotes are doubled and a cell is wrapped when it holds a comma, a //! quote or a line break, which is the rule everything that reads CSV agrees on. +//! +//! Read back, a cell is its value as written: `*literal*` in a spreadsheet is those nine +//! characters, not a word in italics. use crate::markdown::{Block, Span}; @@ -11,7 +14,7 @@ pub fn from_table(block: &Block) -> Option { let Block::Table { head, rows } = block else { return None }; let mut out = String::new(); for row in std::iter::once(head).chain(rows.iter()) { - let line: Vec = row.iter().map(|cell| quote(&plain(cell))).collect(); + let line: Vec = row.iter().map(|cell| quote(&plain(cell), row.len() == 1)).collect(); out.push_str(&line.join(",")); out.push('\n'); } @@ -23,8 +26,14 @@ fn plain(spans: &[Span]) -> String { spans.iter().map(|s| s.text.as_str()).collect() } -fn quote(cell: &str) -> String { - if cell.contains([',', '"', '\n', '\r']) { format!("\"{}\"", cell.replace('"', "\"\"")) } else { cell.to_string() } +/// The cell as CSV. The only cell of a row is wrapped when it is blank too: written bare, its +/// line would be a blank line, which a reader skips, and the rows under it would move up. +fn quote(cell: &str, alone: bool) -> String { + if cell.contains([',', '"', '\n', '\r']) || (alone && cell.trim().is_empty()) { + format!("\"{}\"", cell.replace('"', "\"\"")) + } else { + cell.to_string() + } } /// A table read back from CSV: the first row is the heading, as it was written out. @@ -33,20 +42,30 @@ fn quote(cell: &str) -> String { /// doubled quote inside one is a quote, and a wrapped cell may hold commas and line /// breaks. Anything else is taken as it stands rather than refused, because a spreadsheet /// that wrote it is not going to be argued with. +/// +/// A line with nothing on it is skipped, but a row whose one cell was written as `""` is a +/// blank row of the table and is kept, so the rows under it stay where they were. pub fn to_table(text: &str) -> Option { let rows = rows_of(text); - let mut rows = rows.into_iter().filter(|r| !(r.len() == 1 && r[0].trim().is_empty())); - let head: Vec> = rows.next()?.iter().map(|c| crate::markdown::parse_inline(c)).collect(); - let body: Vec>> = rows.map(|r| r.iter().map(|c| crate::markdown::parse_inline(c)).collect()).collect(); + let mut rows = rows.into_iter().filter(|(r, quoted)| *quoted || !(r.len() == 1 && r[0].trim().is_empty())).map(|(r, _)| r); + let head: Vec> = rows.next()?.iter().map(|c| literal(c)).collect(); + let body: Vec>> = rows.map(|r| r.iter().map(|c| literal(c)).collect()).collect(); Some(Block::Table { head, rows: body }) } -/// The cells of each line, with quotes undone. -fn rows_of(text: &str) -> Vec> { +/// A cell's value as plain text. A spreadsheet cell holds a value, so its asterisks, +/// brackets and backticks are characters of that value, not marks for the page. +fn literal(cell: &str) -> Vec { + if cell.is_empty() { Vec::new() } else { vec![Span { text: cell.to_string(), ..Span::default() }] } +} + +/// The cells of each line, with quotes undone, and whether any cell on the line was quoted. +fn rows_of(text: &str) -> Vec<(Vec, bool)> { let mut rows = Vec::new(); let mut row: Vec = Vec::new(); let mut cell = String::new(); let mut quoted = false; + let mut any_quoted = false; let mut chars = text.chars().peekable(); while let Some(c) = chars.next() { match c { @@ -61,19 +80,20 @@ fn rows_of(text: &str) -> Vec> { '"' if cell.trim().is_empty() => { cell.clear(); quoted = true; + any_quoted = true; } ',' if !quoted => row.push(std::mem::take(&mut cell)), '\r' if !quoted => {} '\n' if !quoted => { row.push(std::mem::take(&mut cell)); - rows.push(std::mem::take(&mut row)); + rows.push((std::mem::take(&mut row), std::mem::take(&mut any_quoted))); } other => cell.push(other), } } - if !cell.is_empty() || !row.is_empty() { + if !cell.is_empty() || !row.is_empty() || any_quoted { row.push(cell); - rows.push(row); + rows.push((row, any_quoted)); } rows } @@ -129,6 +149,37 @@ lines", assert_eq!(text(&rows[1][1]), "last"); } + /// Issue #110: a cell is read as the value it holds. Read as Markdown, `*literal*` came + /// back as an italic `literal` and `[a](b)` as a link, and writing the table out again + /// lost the characters. + #[test] + fn a_cell_that_looks_like_markdown_is_read_as_it_is_written() { + for input in ["Value\n*literal*\n", "Value\n[a](b)\n", "Value\n`x` **y** _z_ w\n", "# Heading,- item\n> quote,1. one\n"] { + let table = to_table(input).expect("a table"); + let Block::Table { head, rows } = &table else { panic!("a table") }; + let plain_only = + std::iter::once(head).chain(rows).flatten().flatten().all(|span| span == &Span { text: span.text.clone(), ..Span::default() }); + assert!(plain_only, "{input:?} is read without marks"); + assert_eq!(from_table(&table).as_deref(), Some(input), "{input:?} comes back character for character"); + } + } + + /// Issue #112: a blank row of a one-column table survives the trip through CSV, and so + /// do the rows under it. + #[test] + fn a_blank_row_of_a_one_column_table_is_kept() { + let cell = |text: &str| if text.is_empty() { Vec::new() } else { vec![Span { text: text.into(), ..Span::default() }] }; + let table = Block::Table { head: vec![cell("Value")], rows: vec![vec![cell("")], vec![cell("value")], vec![cell(" ")]] }; + let csv = from_table(&table).expect("a table"); + assert_eq!(csv, "Value\n\"\"\nvalue\n\" \"\n", "a blank only cell is written quoted"); + assert_eq!(to_table(&csv), Some(table), "and read back as a row"); + + // What a spreadsheet writes for a blank row, with blank lines between the rows that + // are not rows. + let Some(Block::Table { rows, .. }) = to_table("Value\n\"\"\n\nvalue\n\n") else { panic!("a table") }; + assert_eq!(rows, vec![vec![cell("")], vec![cell("value")]]); + } + #[test] fn nothing_in_nothing_out() { assert_eq!(to_table(""), None); diff --git a/crates/formats/src/html.rs b/crates/formats/src/html.rs index 2d5b7455..37844d0b 100644 --- a/crates/formats/src/html.rs +++ b/crates/formats/src/html.rs @@ -110,18 +110,39 @@ pub fn unescape(s: &str) -> String { out.replace("&", "&") } -/// Plain text of an HTML fragment: tags removed, entities decoded. +/// Tags that mark words inside a line. They join what is on either side of them, so the +/// letters of `format` stay one word; every other tag ends a word. +const INLINE_TAGS: &[&str] = &[ + "a", "abbr", "b", "bdi", "bdo", "big", "cite", "code", "del", "dfn", "em", "font", "i", "ins", "kbd", "mark", "q", "s", "samp", "small", "span", + "strike", "strong", "sub", "sup", "time", "tt", "u", "var", "wbr", +]; + +/// Plain text of an HTML fragment: tags removed, entities decoded. A tag that starts or ends +/// a block, a line break or a cell leaves a space behind; formatting inside a line leaves +/// nothing, so a word that is partly bold is still one word (issue #105). pub fn strip_tags(html: &str) -> String { - let (mut out, mut in_tag) = (String::new(), false); + let (mut out, mut tag, mut in_tag) = (String::new(), String::new(), false); for c in html.chars() { match c { - '<' => in_tag = true, - '>' => { + '<' => { + in_tag = true; + tag.clear(); + } + '>' if in_tag => { in_tag = false; - out.push(' '); + let name: String = tag + .trim_start() + .trim_start_matches('/') + .chars() + .take_while(|c| c.is_ascii_alphanumeric()) + .collect::() + .to_ascii_lowercase(); + if !INLINE_TAGS.contains(&name.as_str()) { + out.push(' '); + } } - c if !in_tag => out.push(c), - _ => {} + c if in_tag => tag.push(c), + c => out.push(c), } } unescape(&out) @@ -323,9 +344,14 @@ struct Writer { lists: Vec<(bool, usize)>, // (ordered, counter) in_pre: bool, in_code: bool, + /// Where the open inline code's text starts, in `cell` inside a table and `out` elsewhere. + code_at: Option, pre_diagram: bool, /// Inside a `
`, which Mermaid's own pages use instead of a `pre`. div_diagram: bool, + /// Where the open code block's fence starts in `out`, so it can be made longer when the + /// code turns out to hold a fence of its own. + fence_at: Option, in_math: bool, in_math_block: bool, span_sizes: Vec, @@ -338,6 +364,26 @@ struct Writer { } impl Writer { + /// Remembers where the fence of a code block about to be written starts. + fn open_fence(&mut self) { + self.fence_at = (!self.in_cell).then_some(self.out.len()); + } + + /// Writes the fence that closes the open code block. When a line of the code starts with + /// three or more backticks, both fences are made longer than that run, so the code stays + /// one block when it is read again (issue #106). + fn close_fence(&mut self) { + let mut fence = "```".to_string(); + if let Some(at) = self.fence_at.take().filter(|&at| self.out.is_char_boundary(at)) { + let code_from = self.out[at..].find('\n').map_or(self.out.len(), |n| at + n + 1); + fence = crate::markdown::code_fence(&self.out[code_from..]); + if fence.len() > 3 { + self.out.insert_str(at, &fence[3..]); + } + } + self.out.push_str(&fence); + } + fn push(&mut self, s: &str) { if self.in_cell { self.cell.push_str(s); @@ -423,6 +469,7 @@ impl Writer { // A div with the class `mermaid` holds a diagram, as a pre does. Its text is the source. ("div", false) if !self.in_pre && attrs.to_lowercase().contains("mermaid") => { self.block_break(); + self.open_fence(); self.push("```mermaid\n"); self.in_pre = true; self.pre_diagram = true; @@ -434,7 +481,7 @@ impl Writer { if !self.out.ends_with('\n') { self.out.push('\n'); } - self.out.push_str("```"); + self.close_fence(); self.block_break(); } ("p", true) if self.in_math_block => { @@ -511,9 +558,23 @@ impl Writer { ("i" | "em", _) => self.push("*"), ("s" | "del" | "strike", _) => self.push("~~"), ("u" | "sub" | "sup", closing) => self.push(&format!("<{}{name}>", if closing { "/" } else { "" })), - ("code", closing) if !self.in_pre => { - self.in_code = !closing; - self.push("`"); + ("code", false) if !self.in_pre => { + self.in_code = true; + self.code_at = Some(if self.in_cell { self.cell.len() } else { self.out.len() }); + } + ("code", true) if !self.in_pre => { + self.in_code = false; + // The code is written with the delimiters it needs once all of it is known: a + // backtick inside it would otherwise end it early (issue #107). + if let Some(at) = self.code_at.take() { + let buffer = if self.in_cell { &mut self.cell } else { &mut self.out }; + if buffer.is_char_boundary(at) { + let code = buffer.split_off(at); + if !code.is_empty() { + buffer.push_str(&crate::render::code_span(&code)); + } + } + } } ("pre", false) => { self.block_break(); @@ -522,6 +583,7 @@ impl Writer { let class = attr(attrs, "class").unwrap_or_default(); self.pre_diagram = class.eq_ignore_ascii_case("diagram") || class.eq_ignore_ascii_case("mermaid"); let language = if self.pre_diagram { "mermaid" } else { class.as_str() }; + self.open_fence(); self.push(&format!("```{language}\n")); self.in_pre = true; } @@ -530,7 +592,7 @@ impl Writer { if !self.out.ends_with('\n') { self.out.push('\n'); } - self.out.push_str("```"); + self.close_fence(); self.block_break(); } ("span", false) => { @@ -565,7 +627,7 @@ impl Writer { } ("a", true) => { let href = self.href.pop().flatten().unwrap_or_default(); - self.push(&format!("]({href})")); + self.push(&format!("]({})", crate::render::destination(&href))); } ("img", _) => { let alt = attr(attrs, "alt").unwrap_or_default(); @@ -576,7 +638,9 @@ impl Writer { // Block-level img with size: output as HTML. Inline or no size: use markdown. if (width.is_some() || height.is_some()) && self.href.is_empty() && !self.in_cell { self.block_break(); - let mut img_tag = format!("\"{alt}\""); { let src = attr(attrs, "src").unwrap_or_default(); - self.push(&format!("![{name}]({src})")); + self.push(&format!("![{name}]({})", crate::render::destination(&src))); } ("table", false) => { self.block_break(); @@ -723,6 +787,19 @@ mod tests { assert_eq!(strip_tags("

Hello big world

").split_whitespace().collect::>().join(" "), "Hello big world"); } + /// Issue #105: formatting inside a word keeps the word whole, and blocks, line breaks + /// and cells still keep their words apart. + #[test] + fn stripping_tags_keeps_a_partly_formatted_word_whole() { + let words = |html: &str| strip_tags(html).split_whitespace().collect::>().join(" "); + assert_eq!(words("

format

"), "format"); + assert_eq!(words("

cooperate link

"), "cooperate link"); + assert_eq!(words("

one

two

"), "one two"); + assert_eq!(words("

one
two
three

"), "one two three"); + assert_eq!(words("
  • a
  • b
cd
"), "a b c d"); + assert_eq!(words("

Title

Body & more"), "Title Body & more"); + } + #[test] fn converts_blocks_and_inline_markup() { let md = to_markdown("

Title

One bold and italic and code.

  • a
  • b
"); diff --git a/crates/formats/src/markdown.rs b/crates/formats/src/markdown.rs index f75dfaa0..edb32167 100644 --- a/crates/formats/src/markdown.rs +++ b/crates/formats/src/markdown.rs @@ -98,12 +98,22 @@ fn find(chars: &[char], from: usize, pat: &[char]) -> Option { None } -// Backslashes stay literal inside code and legacy link destinations. -fn find_raw(chars: &[char], from: usize, pat: &[char]) -> Option { - if pat.is_empty() || chars.len() < pat.len() { - return None; +/// Where the run of exactly `run` backticks that closes a code span starts, looking from +/// `from`. A longer or shorter run is part of the code. +fn closing_ticks(chars: &[char], from: usize, run: usize) -> Option { + let mut i = from; + while i < chars.len() { + if chars[i] == '`' { + let len = chars[i..].iter().take_while(|&&c| c == '`').count(); + if len == run { + return Some(i); + } + i += len; + } else { + i += 1; + } } - (from..=chars.len() - pat.len()).find(|&i| chars.get(i..i + pat.len()) == Some(pat)) + None } fn slice(chars: &[char], a: usize, b: usize) -> &[char] { @@ -327,9 +337,19 @@ fn inline(chars: &[char], st: Style, link: Option, depth: usize) -> Vec< i += 2; } '`' => { - if let Some(end) = find_raw(chars, i + 1, &['`']) { + // A code span opens with a run of backticks and closes at the next run of + // exactly as many, so a backtick inside it is code (issue #107). A run with no + // partner is literal text. + let run = chars[i..].iter().take_while(|&&c| c == '`').count(); + if let Some(end) = closing_ticks(chars, i + run, run) { push_text(&mut out, &mut buf, &st, &link); - let code: String = slice(chars, i + 1, end).iter().collect(); + let code: String = slice(chars, i + run, end).iter().collect(); + // One space each side is padding when both are there, as CommonMark has it, + // so a span can start or end with a backtick. + let code = match code.strip_prefix(' ').and_then(|c| c.strip_suffix(' ')) { + Some(inner) if !code.chars().all(|c| c == ' ') => inner.to_string(), + _ => code, + }; out.push(Span { text: code, code: true, @@ -343,10 +363,10 @@ fn inline(chars: &[char], st: Style, link: Option, depth: usize) -> Vec< link: link.clone(), ..Span::default() }); - i = end + 1; + i = end + run; } else { - buf.push(c); - i += 1; + buf.extend(std::iter::repeat_n('`', run)); + i += run; } } // `***x***` is bold and italic together, which is how both marks on one run are written. @@ -490,13 +510,41 @@ fn parse_link(chars: &[char], start: usize) -> Option<(String, String, usize)> { if chars.get(close + 1) != Some(&'(') { return None; } - let end = find_raw(chars, close + 2, &[')'])?; let label: String = slice(chars, start + 1, close).iter().collect(); - let url: String = slice(chars, close + 2, end).iter().collect::().trim().to_string(); - if url.contains(char::is_whitespace) { - return None; + // Read first with `\(` and `\)` as quoted parentheses. A link written before quoting + // existed may end in a Windows folder, `C:\Notes\)`, whose last backslash is part of the + // path: when the quoted reading finds no end, or one that takes in words after the link, + // the backslashes are read as they stand. + for quoted in [true, false] { + let Some(end) = destination_end(chars, close + 2, quoted) else { continue }; + let url: String = slice(chars, close + 2, end).iter().collect::().trim().to_string(); + if url.contains(char::is_whitespace) { + continue; + } + let url = if quoted { url.replace("\\(", "(").replace("\\)", ")") } else { url }; + return Some((label, url, end + 1)); } - Some((label, url, end + 1)) + None +} + +/// The `)` that ends a link's destination starting at `from`. Parentheses inside it nest, as +/// in `a(b).pdf`, and with `quoted`, `\(` or `\)` is a parenthesis that does not count, so +/// the first `)` is not taken for the end (issue #108). +fn destination_end(chars: &[char], from: usize, quoted: bool) -> Option { + let mut depth = 0usize; + let mut i = from; + while let Some(&c) = chars.get(i) { + match c { + '\\' if quoted && matches!(chars.get(i + 1), Some('(' | ')')) => i += 1, + '(' if depth < 32 => depth += 1, + '(' => return None, + ')' if depth == 0 => return Some(i), + ')' => depth -= 1, + _ => {} + } + i += 1; + } + None } fn is_table_separator(line: &str) -> bool { @@ -558,7 +606,8 @@ fn parse_attr(tag: &str, name: &str) -> Option { let marker = format!("{name}=\""); let start = lower.find(&marker)? + marker.len(); let end = tag.get(start..)?.find('"')?; - tag.get(start..start + end).map(str::to_string) + // The value as it was before it was escaped for the tag: `"` is a quote in the text. + tag.get(start..start + end).map(crate::html::unescape) } fn try_parse_img_block(line: &str) -> Option { @@ -573,6 +622,18 @@ fn try_parse_img_block(line: &str) -> Option { Some(Block::Image { alt, src, width, height }) } +/// How many backticks `text` starts with. +fn backtick_run(text: &str) -> usize { + text.bytes().take_while(|&b| b == b'`').count() +} + +/// The fence to write round `code`: three backticks, or one more than the longest run that +/// starts a line of the code, so no line of it can close the block early (issue #106). +pub(crate) fn code_fence(code: &str) -> String { + let longest = code.lines().map(|line| backtick_run(line.trim_start())).max().unwrap_or(0); + "`".repeat(longest.max(2) + 1) +} + /// Parses a Markdown page body into blocks. Never panics. pub fn parse(src: &str) -> Vec { let text = src.replace("\r\n", "\n"); @@ -584,11 +645,19 @@ pub fn parse(src: &str) -> Vec { let t = line.trim(); if t.is_empty() { i += 1; - } else if let Some(fence) = t.strip_prefix("```") { - let lang = fence.trim().to_string(); + } else if t.starts_with("```") { + // The fence is as long as its run of backticks, and only a run at least as long, + // with nothing after it, closes it: a shorter fence inside is part of the code. + let open = backtick_run(t); + let lang = t[open..].trim().to_string(); + let closes = |l: &str| { + let l = l.trim(); + let run = backtick_run(l); + run >= open && l[run..].is_empty() + }; let mut body = Vec::new(); i += 1; - while i < lines.len() && !lines.get(i).is_some_and(|l| l.trim_start().starts_with("```")) { + while i < lines.len() && !lines.get(i).is_some_and(|l| closes(l)) { body.push(lines.get(i).copied().unwrap_or("")); i += 1; } diff --git a/crates/formats/src/render.rs b/crates/formats/src/render.rs index 446f9b37..696134b2 100644 --- a/crates/formats/src/render.rs +++ b/crates/formats/src/render.rs @@ -48,14 +48,16 @@ pub fn blocks_to_markdown(blocks: &[Block]) -> String { out.push('\n'); } Block::Code { lang, text } => { - out.push_str("```"); + let fence = crate::markdown::code_fence(text); + out.push_str(&fence); out.push_str(lang); out.push('\n'); out.push_str(text); if !text.ends_with('\n') { out.push('\n'); } - out.push_str("```\n"); + out.push_str(&fence); + out.push('\n'); } Block::MathBlock(tex) => { out.push_str("$$\n"); @@ -72,7 +74,8 @@ pub fn blocks_to_markdown(blocks: &[Block]) -> String { Block::Rule => out.push_str("---\n"), Block::Image { alt, src, width, height } => { if width.is_some() || height.is_some() { - let mut img_tag = format!("\"{alt}\""); String { img_tag.push_str(">\n"); out.push_str(&img_tag); } else { - out.push_str(&format!("![{alt}]({src})\n")); + out.push_str(&format!("![{alt}]({})\n", destination(src))); } } } @@ -234,12 +237,28 @@ fn marked(spans: &[Span], level: usize, literal: &Literal) -> String { } /// One span with nothing but its code, picture, maths or link written round it. +/// Inline code written so it reads back as the same text (issue #107). The delimiter is one +/// backtick more than the longest run inside, and a space pads each side when the code starts +/// or ends with a backtick, or starts and ends with a space, which the reader takes off again. +pub(crate) fn code_span(text: &str) -> String { + let mut longest = 0; + let mut run = 0; + for c in text.chars() { + run = if c == '`' { run + 1 } else { 0 }; + longest = longest.max(run); + } + let ticks = "`".repeat(longest + 1); + let pad = text.starts_with('`') || text.ends_with('`') || (text.starts_with(' ') && text.ends_with(' ') && text.trim() != ""); + let pad = if pad { " " } else { "" }; + format!("{ticks}{pad}{text}{pad}{ticks}") +} + fn leaf(s: &Span, literal: &Literal) -> String { let styled = |text: String| inline_style(s, inline_marks(s, text)); if let Some(src) = &s.image { - let image = format!("![{}]({src})", s.text); + let image = format!("![{}]({})", s.text, destination(src)); let image = match &s.link { - Some(target) => format!("[{image}]({target})"), + Some(target) => format!("[{image}]({})", destination(target)), None => image, }; return styled(image); @@ -247,14 +266,30 @@ fn leaf(s: &Span, literal: &Literal) -> String { if s.math { return styled(format!("${}$", s.text)); } - let text = if s.code { format!("`{}`", s.text) } else { literal.escape(&s.text) }; + let text = if s.code { code_span(&s.text) } else { literal.escape(&s.text) }; let text = match &s.link { - Some(target) => format!("[{text}]({target})"), + Some(target) => format!("[{text}]({})", destination(target)), None => text, }; styled(text) } +/// A link or picture address as it is written between `(` and `)`. Balanced parentheses are +/// left as they are, as in `a(b).pdf`; when they do not pair up, each is written `\(` or `\)` +/// so the reader does not end the address at the wrong one (issue #108). +pub(crate) fn destination(target: &str) -> String { + let mut depth = 0i32; + let balanced = target.chars().all(|c| { + match c { + '(' => depth += 1, + ')' => depth -= 1, + _ => {} + } + depth >= 0 + }) && depth == 0; + if balanced { target.to_string() } else { target.replace('(', "\\(").replace(')', "\\)") } +} + /// Which delimiters a run of spans must quote so its words read back as words. /// /// The inline parser only treats `\\` before punctuation, `` ` ``, `*`, `~~`, `$`, `[`/`]` and `![` @@ -645,6 +680,142 @@ fn b() {}" } } + /// Issue #106: code that holds a fence line of its own stays one block, in both formats. + /// The fence written round it is longer than any run of backticks starting one of its + /// lines, and only a fence at least as long as the opening one closes the block. + #[test] + fn code_holding_a_fence_stays_one_block() { + let tick = "`"; + for text in [ + format!("before\n{}\nafter", tick.repeat(3)), + format!("{}md\nnested\n{}", tick.repeat(3), tick.repeat(3)), + format!(" {}\n{}python\nx\n{}", tick.repeat(5), tick.repeat(4), tick.repeat(4)), + format!("inline {} only", tick.repeat(3)), + ] { + let original = vec![Block::Code { lang: "markdown".into(), text: text.clone() }]; + let saved = blocks_to_markdown(&original); + assert_eq!(parse(&saved), original, "CMGFM:\n{saved}"); + let html = blocks_to_html(&original); + let reread = html::to_markdown(&html); + assert_eq!(parse(&reread), original, "HTML:\n{html}\n--- read as ---\n{reread}"); + } + // A plain block keeps the three backticks it always had. + assert_eq!( + blocks_to_markdown(&[Block::Code { lang: "rust".into(), text: "let x = 1;".into() }]), + format!("{0}rust\nlet x = 1;\n{0}\n", tick.repeat(3)) + ); + // A shorter fence inside a longer one is code, and a closing fence with words after + // it is not a closing fence. + let four = tick.repeat(4); + let three = tick.repeat(3); + assert_eq!( + parse(&format!("{four}\n{three}\n{three}js\n{four}\n")), + vec![Block::Code { lang: String::new(), text: format!("{three}\n{three}js") }] + ); + } + + /// Issue #107: inline code holding backticks, or starting or ending with one or with a + /// space, comes back as the same single span in both formats. + #[test] + fn inline_code_holding_backticks_stays_one_span() { + let tick = "`"; + for text in [ + format!("a{tick}b"), + format!("a{0}{0}b{0}c", tick), + format!("{tick}start"), + format!("end{tick}"), + tick.to_string(), + " padded ".to_string(), + " two ".to_string(), + " ".to_string(), + "plain".to_string(), + ] { + let original = vec![Block::Paragraph(vec![ + Span { text: "see ".into(), ..Span::default() }, + Span { text: text.clone(), code: true, ..Span::default() }, + Span { text: " here".into(), ..Span::default() }, + ])]; + let saved = blocks_to_markdown(&original); + assert_eq!(parse(&saved), original, "CMGFM {text:?}:\n{saved}"); + // HTML folds a space at the edge of the code into the space before or after it, as a + // browser shows it, so those are only kept in CMGFM. + if text.starts_with(' ') || text.ends_with(' ') { + continue; + } + let html = blocks_to_html(&original); + let reread = html::to_markdown(&html); + assert_eq!(parse(&reread), original, "HTML {text:?}:\n{html}\n--- read as ---\n{reread}"); + } + // Plain code keeps its one backtick each side. + assert_eq!( + blocks_to_markdown(&[Block::Paragraph(vec![Span { text: "x".into(), code: true, ..Span::default() }])]), + format!("{tick}x{tick}\n") + ); + // A run with no partner of the same length is text. + let lone = parse(&format!("a {0}{0} b {0} c\n", tick)); + assert!(matches!(&lone[..], [Block::Paragraph(spans)] if spans.iter().all(|s| !s.code)), "{lone:?}"); + } + + /// Issue #108: a link or picture address with parentheses in it is kept whole, in both + /// formats, whether its parentheses pair up or not. + #[test] + fn link_addresses_with_parentheses_are_kept_whole() { + let parsed = parse("[file](https://example.test/a(b).pdf)\n"); + let link = |target: &str| vec![Block::Paragraph(vec![Span { text: "file".into(), link: Some(target.into()), ..Span::default() }])]; + assert_eq!(parsed, link("https://example.test/a(b).pdf"), "a balanced pair stays in the address"); + assert_eq!( + parse("[file](a\\(b.pdf) after\n")[0], + Block::Paragraph(vec![ + Span { text: "file".into(), link: Some("a(b.pdf".into()), ..Span::default() }, + Span { text: " after".into(), ..Span::default() }, + ]), + "an escaped parenthesis does not count" + ); + + for target in ["https://example.test/a(b).pdf", "assets/a(b).pdf", "x/(a(b)c).md", "open(.pdf", "close).pdf", ")(", "w/Foo_(bar)"] { + let original = link(target); + let saved = blocks_to_markdown(&original); + assert_eq!(parse(&saved), original, "CMGFM {target:?}:\n{saved}"); + let html = blocks_to_html(&original); + let reread = html::to_markdown(&html); + assert_eq!(parse(&reread), original, "HTML {target:?}:\n{html}\n--- read as ---\n{reread}"); + + let picture = vec![Block::Image { alt: "pic".into(), src: target.into(), width: None, height: None }]; + assert_eq!(parse(&blocks_to_markdown(&picture)), picture, "picture {target:?}"); + assert_eq!(parse(&html::to_markdown(&blocks_to_html(&picture))), picture, "HTML picture {target:?}"); + } + assert_eq!(blocks_to_markdown(&link("a(b).pdf")), "[file](a(b).pdf)\n", "balanced parentheses are written as they are"); + + // A link written before quoting existed, to a Windows folder, keeps its last backslash, + // even with a parenthesis later in the line. + assert_eq!(parse("[file](C:\\Notes\\)\n"), link("C:\\Notes\\")); + assert_eq!( + parse("[file](C:\\Notes\\) and (more) text\n")[0], + Block::Paragraph(vec![ + Span { text: "file".into(), link: Some("C:\\Notes\\".into()), ..Span::default() }, + Span { text: " and (more) text".into(), ..Span::default() }, + ]) + ); + } + + /// Issue #109: a resized picture whose alt text or address holds a quote, an ampersand or + /// an angle bracket keeps all of it, in both formats. + #[test] + fn a_resized_picture_keeps_quotes_in_its_alt_text() { + for (alt, src) in + [("a\"b", "assets/a.png"), ("Tom & \"Jerry\" <3", "assets/t&j.png"), ("it's", "assets/q\"uote.png"), ("& literal", "assets/a.png")] + { + let original = vec![Block::Image { alt: alt.into(), src: src.into(), width: Some(100.0), height: Some(80.0) }]; + let md = blocks_to_markdown(&original); + assert_eq!(parse(&md), original, "CMGFM:\n{md}"); + let html = blocks_to_html(&original); + let reread = html::to_markdown(&html); + assert_eq!(parse(&reread), original, "HTML:\n{html}\n--- read as ---\n{reread}"); + } + let md = blocks_to_markdown(&[Block::Image { alt: "a\"b".into(), src: "assets/a.png".into(), width: Some(100.0), height: None }]); + assert_eq!(md, "\"a"b\"\n"); + } + #[test] fn html_escapes_everything_that_matters() { let blocks = parse("A