From e272062377ca8fe73fbeb150745681d5852d2d6d Mon Sep 17 00:00:00 2001 From: DWAA1660 Date: Thu, 12 Nov 2026 10:12:35 -0700 Subject: [PATCH 01/26] fix(store): resolve links whose first bytes end inside a character MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `resolve_in` checked for the `notecraft:` scheme by slicing the link's first ten bytes. A link such as `../日本語.txt` has a character across byte 10, so the slice panicked, and a Ctrl+click on such a link took the window down (#102). The prefix is now read with `str::get`, which returns nothing there instead of panicking. Fixes #102 Co-Authored-By: Claude Opus 5.5 --- crates/store/src/links.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/crates/store/src/links.rs b/crates/store/src/links.rs index bf61559f..36ef18ae 100644 --- a/crates/store/src/links.rs +++ b/crates/store/src/links.rs @@ -67,8 +67,10 @@ pub fn resolve_in(roots: &[PathBuf], page_dir: &Path, href: &str) -> Link { } // A link of our own: the path after the scheme is read from each open notebook's // folder, so the same link works from any page and in any notebook that holds it. - if href.len() >= SCHEME.len() && href[..SCHEME.len()].eq_ignore_ascii_case(SCHEME) { - let rest = href[SCHEME.len()..].trim_start_matches('/'); + // `get` rather than slicing: the first bytes of a link such as `../日本語.txt` end inside a + // character, and a slice there panics (issue #102). + if let Some(rest) = href.get(..SCHEME.len()).filter(|head| head.eq_ignore_ascii_case(SCHEME)).map(|_| &href[SCHEME.len()..]) { + let rest = rest.trim_start_matches('/'); let rest = rest.split(['#', '?']).next().unwrap_or(""); let decoded = percent_decode(rest); if decoded.is_empty() || decoded.contains('\\') || decoded.contains('\0') { @@ -325,6 +327,20 @@ mod tests { assert_eq!(resolve(&root, &a, ""), Link::Missing(a)); } + /// Issue #102: a link whose first bytes end inside a character panicked on the check for + /// `notecraft:`. Each of these has a character across byte 10, the length of the scheme. + #[test] + fn a_link_with_characters_across_the_scheme_length_does_not_panic() { + let (_t, root, a, _b) = fixture("links-unicode"); + for href in ["../日本語.txt", "日本語語.txt", "aééééé.txt", "notecraf日本"] { + assert!(href.len() > SCHEME.len() && !href.is_char_boundary(SCHEME.len()), "{href} must straddle the scheme length"); + assert!(matches!(resolve(&root, &a, href), Link::Missing(_) | Link::Outside), "{href}"); + assert!(matches!(resolve_in(std::slice::from_ref(&root), &a, href), Link::Missing(_) | Link::Outside), "{href}"); + } + std::fs::write(a.join("assets").join("日本語.txt"), b"text").expect("a file with a Japanese name"); + assert_eq!(resolve(&root, &a, "assets/日本語.txt"), Link::File(a.join("assets").join("日本語.txt"))); + } + #[test] fn percent_escapes_decode_and_junk_is_kept() { assert_eq!(percent_decode("a%20b"), "a b"); From 83a639cf4b72205a7feb69c9c683296b1aaaf175 Mon Sep 17 00:00:00 2001 From: DWAA1660 Date: Thu, 12 Nov 2026 10:15:02 -0700 Subject: [PATCH 02/26] test(ui-egui): order the saver coalescing test with channels, no sleep `saves_asked_for_meanwhile_write_only_the_newest_text` slept 50 ms after the first save and assumed the worker had picked it up by then. Under a loaded parallel run it sometimes had not, so all three saves were rightly made as one and the test failed (#116). The first write now says when it has begun and holds until the later saves are queued, so the interleaving is certain. The test also checks that the second write answers for both saves and nothing is left in flight. Fixes #116 Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/saver.rs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/crates/ui-egui/src/saver.rs b/crates/ui-egui/src/saver.rs index 61d5b01e..930fbea2 100644 --- a/crates/ui-egui/src/saver.rs +++ b/crates/ui-egui/src/saver.rs @@ -151,17 +151,35 @@ mod tests { } /// Saves of one page asked for while it is being written become one, of the newest text. + /// + /// The first write says when it has begun and then holds until the later saves have been + /// asked for, so the order is certain. A fixed sleep did not promise that the worker had + /// picked up the first save yet; when it had not, all three were rightly made as one and + /// the test failed (issue #116). #[test] fn saves_asked_for_meanwhile_write_only_the_newest_text() { let wrote = Arc::new(Mutex::new(Vec::new())); - let mut saver = Saver::with_writer(slow(wrote.clone())); + let (began, writing) = std::sync::mpsc::channel::<()>(); + let (release, held) = std::sync::mpsc::channel::<()>(); + let gate = Mutex::new(Some((began, held))); + let slow = slow(wrote.clone()); + let mut saver = Saver::with_writer(move |dir, body| { + if let Some((began, held)) = gate.lock().expect("lock").take() { + let _ = began.send(()); + let _ = held.recv_timeout(Duration::from_secs(10)); + } + slow(dir, body) + }); saver.submit(Path::new("page"), "one"); - std::thread::sleep(Duration::from_millis(50)); + writing.recv_timeout(Duration::from_secs(10)).expect("the first save began"); saver.submit(Path::new("page"), "two"); saver.submit(Path::new("page"), "three"); - saver.wait(); + release.send(()).expect("let the first save finish"); + let done = saver.wait(); let wrote = wrote.lock().expect("lock").clone(); assert_eq!(wrote, vec!["one".to_string(), "three".to_string()], "the newest text is written last, and only once"); + assert_eq!(done.iter().map(|saved| saved.jobs).collect::>(), vec![1, 2], "the second write answers for both saves"); + assert!(!saver.busy(), "and nothing is left to write"); } /// A save that fails says why. From 7e9e51cdb07f04cd9a3cccc2925738bdbfd9694c Mon Sep 17 00:00:00 2001 From: DWAA1660 Date: Thu, 12 Nov 2026 10:19:13 -0700 Subject: [PATCH 03/26] fix(formats): read CSV cells as literal values, not Markdown `csv::to_table` ran every cell through the inline Markdown parser, so a spreadsheet value such as `*literal*` came back as an italic `literal` and `[a](b)` as a link. Writing the table out again lost the characters (#110). A cell is now one plain span holding its value as written. Fixes #110 Co-Authored-By: Claude Opus 5.5 --- crates/formats/src/csv.rs | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/crates/formats/src/csv.rs b/crates/formats/src/csv.rs index 71db9543..a10848e2 100644 --- a/crates/formats/src/csv.rs +++ b/crates/formats/src/csv.rs @@ -3,6 +3,9 @@ //! The marks inside a cell are dropped, because a spreadsheet cell holds a value, not a //! marked-up line. Quotes are doubled and a cell is wrapped when it holds a comma, a //! quote or a line break, which is the rule everything that reads CSV agrees on. +//! +//! Read back, a cell is its value as written: `*literal*` in a spreadsheet is those nine +//! characters, not a word in italics. use crate::markdown::{Block, Span}; @@ -36,11 +39,17 @@ fn quote(cell: &str) -> String { pub fn to_table(text: &str) -> Option { let rows = rows_of(text); let mut rows = rows.into_iter().filter(|r| !(r.len() == 1 && r[0].trim().is_empty())); - let head: Vec> = rows.next()?.iter().map(|c| crate::markdown::parse_inline(c)).collect(); - let body: Vec>> = rows.map(|r| r.iter().map(|c| crate::markdown::parse_inline(c)).collect()).collect(); + let head: Vec> = rows.next()?.iter().map(|c| literal(c)).collect(); + let body: Vec>> = rows.map(|r| r.iter().map(|c| literal(c)).collect()).collect(); Some(Block::Table { head, rows: body }) } +/// A cell's value as plain text. A spreadsheet cell holds a value, so its asterisks, +/// brackets and backticks are characters of that value, not marks for the page. +fn literal(cell: &str) -> Vec { + if cell.is_empty() { Vec::new() } else { vec![Span { text: cell.to_string(), ..Span::default() }] } +} + /// The cells of each line, with quotes undone. fn rows_of(text: &str) -> Vec> { let mut rows = Vec::new(); @@ -129,6 +138,21 @@ lines", assert_eq!(text(&rows[1][1]), "last"); } + /// Issue #110: a cell is read as the value it holds. Read as Markdown, `*literal*` came + /// back as an italic `literal` and `[a](b)` as a link, and writing the table out again + /// lost the characters. + #[test] + fn a_cell_that_looks_like_markdown_is_read_as_it_is_written() { + for input in ["Value\n*literal*\n", "Value\n[a](b)\n", "Value\n`x` **y** _z_ w\n", "# Heading,- item\n> quote,1. one\n"] { + let table = to_table(input).expect("a table"); + let Block::Table { head, rows } = &table else { panic!("a table") }; + let plain_only = + std::iter::once(head).chain(rows).flatten().flatten().all(|span| span == &Span { text: span.text.clone(), ..Span::default() }); + assert!(plain_only, "{input:?} is read without marks"); + assert_eq!(from_table(&table).as_deref(), Some(input), "{input:?} comes back character for character"); + } + } + #[test] fn nothing_in_nothing_out() { assert_eq!(to_table(""), None); From ff8413bdfb6a1c321e3a99cccd0d2d4de309a356 Mon Sep 17 00:00:00 2001 From: DWAA1660 Date: Thu, 12 Nov 2026 10:19:33 -0700 Subject: [PATCH 04/26] fix(formats): keep blank rows of a one-column table in CSV The CSV reader skipped every line whose one cell was blank, which also dropped a blank row of a one-column table, so every row under it moved up (#112). The writer now quotes a blank cell that is alone on its row, and the reader keeps a line that held a quoted cell. Lines with nothing on them are still skipped. Fixes #112 Co-Authored-By: Claude Opus 5.5 --- crates/formats/src/csv.rs | 45 +++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/crates/formats/src/csv.rs b/crates/formats/src/csv.rs index a10848e2..6b68bb11 100644 --- a/crates/formats/src/csv.rs +++ b/crates/formats/src/csv.rs @@ -14,7 +14,7 @@ pub fn from_table(block: &Block) -> Option { let Block::Table { head, rows } = block else { return None }; let mut out = String::new(); for row in std::iter::once(head).chain(rows.iter()) { - let line: Vec = row.iter().map(|cell| quote(&plain(cell))).collect(); + let line: Vec = row.iter().map(|cell| quote(&plain(cell), row.len() == 1)).collect(); out.push_str(&line.join(",")); out.push('\n'); } @@ -26,8 +26,14 @@ fn plain(spans: &[Span]) -> String { spans.iter().map(|s| s.text.as_str()).collect() } -fn quote(cell: &str) -> String { - if cell.contains([',', '"', '\n', '\r']) { format!("\"{}\"", cell.replace('"', "\"\"")) } else { cell.to_string() } +/// The cell as CSV. The only cell of a row is wrapped when it is blank too: written bare, its +/// line would be a blank line, which a reader skips, and the rows under it would move up. +fn quote(cell: &str, alone: bool) -> String { + if cell.contains([',', '"', '\n', '\r']) || (alone && cell.trim().is_empty()) { + format!("\"{}\"", cell.replace('"', "\"\"")) + } else { + cell.to_string() + } } /// A table read back from CSV: the first row is the heading, as it was written out. @@ -36,9 +42,12 @@ fn quote(cell: &str) -> String { /// doubled quote inside one is a quote, and a wrapped cell may hold commas and line /// breaks. Anything else is taken as it stands rather than refused, because a spreadsheet /// that wrote it is not going to be argued with. +/// +/// A line with nothing on it is skipped, but a row whose one cell was written as `""` is a +/// blank row of the table and is kept, so the rows under it stay where they were. pub fn to_table(text: &str) -> Option { let rows = rows_of(text); - let mut rows = rows.into_iter().filter(|r| !(r.len() == 1 && r[0].trim().is_empty())); + let mut rows = rows.into_iter().filter(|(r, quoted)| *quoted || !(r.len() == 1 && r[0].trim().is_empty())).map(|(r, _)| r); let head: Vec> = rows.next()?.iter().map(|c| literal(c)).collect(); let body: Vec>> = rows.map(|r| r.iter().map(|c| literal(c)).collect()).collect(); Some(Block::Table { head, rows: body }) @@ -50,12 +59,13 @@ fn literal(cell: &str) -> Vec { if cell.is_empty() { Vec::new() } else { vec![Span { text: cell.to_string(), ..Span::default() }] } } -/// The cells of each line, with quotes undone. -fn rows_of(text: &str) -> Vec> { +/// The cells of each line, with quotes undone, and whether any cell on the line was quoted. +fn rows_of(text: &str) -> Vec<(Vec, bool)> { let mut rows = Vec::new(); let mut row: Vec = Vec::new(); let mut cell = String::new(); let mut quoted = false; + let mut any_quoted = false; let mut chars = text.chars().peekable(); while let Some(c) = chars.next() { match c { @@ -70,19 +80,20 @@ fn rows_of(text: &str) -> Vec> { '"' if cell.trim().is_empty() => { cell.clear(); quoted = true; + any_quoted = true; } ',' if !quoted => row.push(std::mem::take(&mut cell)), '\r' if !quoted => {} '\n' if !quoted => { row.push(std::mem::take(&mut cell)); - rows.push(std::mem::take(&mut row)); + rows.push((std::mem::take(&mut row), std::mem::take(&mut any_quoted))); } other => cell.push(other), } } - if !cell.is_empty() || !row.is_empty() { + if !cell.is_empty() || !row.is_empty() || any_quoted { row.push(cell); - rows.push(row); + rows.push((row, any_quoted)); } rows } @@ -153,6 +164,22 @@ lines", } } + /// Issue #112: a blank row of a one-column table survives the trip through CSV, and so + /// do the rows under it. + #[test] + fn a_blank_row_of_a_one_column_table_is_kept() { + let cell = |text: &str| if text.is_empty() { Vec::new() } else { vec![Span { text: text.into(), ..Span::default() }] }; + let table = Block::Table { head: vec![cell("Value")], rows: vec![vec![cell("")], vec![cell("value")], vec![cell(" ")]] }; + let csv = from_table(&table).expect("a table"); + assert_eq!(csv, "Value\n\"\"\nvalue\n\" \"\n", "a blank only cell is written quoted"); + assert_eq!(to_table(&csv), Some(table), "and read back as a row"); + + // What a spreadsheet writes for a blank row, with blank lines between the rows that + // are not rows. + let Some(Block::Table { rows, .. }) = to_table("Value\n\"\"\n\nvalue\n\n") else { panic!("a table") }; + assert_eq!(rows, vec![vec![cell("")], vec![cell("value")]]); + } + #[test] fn nothing_in_nothing_out() { assert_eq!(to_table(""), None); From 1ee909109db45bec746cf7e43b151e114fcca1cc Mon Sep 17 00:00:00 2001 From: DWAA1660 Date: Thu, 12 Nov 2026 10:20:53 -0700 Subject: [PATCH 05/26] fix(store): refuse a page file too large to open again `save_body` capped only the body at MAX_PAGE_BYTES, while `load_page` caps the whole file. A body at the limit was saved, but the head round it pushed the file over, so the page could not be opened again (#113). The cap is now on the whole page file in `write_atomic`, which every writer of a page file goes through, and the save is refused before the current file is replaced. Fixes #113 Co-Authored-By: Claude Opus 5.5 --- crates/store/src/page.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/crates/store/src/page.rs b/crates/store/src/page.rs index 86cf71f2..aae24927 100644 --- a/crates/store/src/page.rs +++ b/crates/store/src/page.rs @@ -492,10 +492,17 @@ pub fn save_body(dir: &Path, body: &str) -> Result { /// Writes a file by writing a temporary file next to it and renaming it over the /// target, so a crash cannot leave half a page. +/// +/// A page file larger than [`MAX_PAGE_BYTES`] is refused before anything is written: it could +/// not be opened again, and the page on disk is kept as it was. The limit is on the whole +/// file, head and all, because that is what [`load_page`] measures (issue #113). pub fn write_atomic(path: &Path, text: &str) -> Result<()> { if let Some(parent) = path.parent() && crate::tree::page_file(parent).is_some_and(|(_, page)| page == path) { + if text.len() > MAX_PAGE_BYTES { + return Err(StoreError::invalid("this page is too large to save")); + } snapshot_current(parent, path, false)?; } write_atomic_file(path, text) @@ -512,6 +519,32 @@ fn write_atomic_file(path: &Path, text: &str) -> Result<()> { #[cfg(test)] mod tests { + /// Issue #113: a body at the limit was saved, but the page file round it was larger than + /// the limit, so the page could not be opened again. A save that would make the file too + /// large is refused and the page is left as it was; one that fits opens again. + #[test] + fn every_page_that_saves_opens_again() { + let t = crate::testing::TempDir::new("page-size-limit"); + let settings = crate::settings::Settings::at(t.path().join("settings.toml")); + let notebook = crate::ops::create_notebook(t.path(), "Notebook", None, &settings).expect("notebook"); + for format in [notecraft_model::PageFormat::Html, notecraft_model::PageFormat::Cmgfm] { + let page = crate::ops::create_node(¬ebook, &format!("Page {}", format.id()), notecraft_model::NodeKind::Page, Some(format), &settings) + .expect("page"); + save_body(&page, "kept").expect("a small save"); + let (_, file) = crate::tree::page_file(&page).expect("the page file"); + let before = fs::read_to_string(&file).expect("the page file"); + + let full = "a".repeat(MAX_PAGE_BYTES); + assert!(save_body(&page, &full).is_err(), "{format:?}: a body at the limit leaves no room for the page round it"); + assert_eq!(fs::read_to_string(&file).expect("the page file"), before, "{format:?}: the refused save changed nothing"); + assert_eq!(load_page(&page).expect("the kept page opens").body.trim(), "kept"); + + let fits = "a".repeat(MAX_PAGE_BYTES - 64 * 1024); + save_body(&page, &fits).expect("a body that fits with its head"); + assert_eq!(load_page(&page).expect("the saved page opens again").body.trim().len(), fits.len(), "{format:?}"); + } + } + #[test] fn page_versions_restore_the_saved_page_file() { let t = crate::testing::TempDir::new("page-versions"); From 49279a47d09ad796176d898f5a47ff1e9dcb4b03 Mon Sep 17 00:00:00 2001 From: DWAA1660 Date: Thu, 12 Nov 2026 10:22:47 -0700 Subject: [PATCH 06/26] fix(formats): keep a partly formatted word whole when stripping tags `html::strip_tags` put a space after every tag, so `format` came out as `for mat` and search could not find `format` on an HTML page (#105). Inline formatting tags now leave nothing behind; block tags, line breaks and cells still separate words. A `>` outside a tag is now kept as text. Fixes #105 Co-Authored-By: Claude Opus 5.5 --- crates/formats/src/html.rs | 48 ++++++++++++++++++++++++++++++++------ crates/store/src/search.rs | 13 +++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/crates/formats/src/html.rs b/crates/formats/src/html.rs index 2d5b7455..5c19ac59 100644 --- a/crates/formats/src/html.rs +++ b/crates/formats/src/html.rs @@ -110,18 +110,39 @@ pub fn unescape(s: &str) -> String { out.replace("&", "&") } -/// Plain text of an HTML fragment: tags removed, entities decoded. +/// Tags that mark words inside a line. They join what is on either side of them, so the +/// letters of `format` stay one word; every other tag ends a word. +const INLINE_TAGS: &[&str] = &[ + "a", "abbr", "b", "bdi", "bdo", "big", "cite", "code", "del", "dfn", "em", "font", "i", "ins", "kbd", "mark", "q", "s", "samp", "small", "span", + "strike", "strong", "sub", "sup", "time", "tt", "u", "var", "wbr", +]; + +/// Plain text of an HTML fragment: tags removed, entities decoded. A tag that starts or ends +/// a block, a line break or a cell leaves a space behind; formatting inside a line leaves +/// nothing, so a word that is partly bold is still one word (issue #105). pub fn strip_tags(html: &str) -> String { - let (mut out, mut in_tag) = (String::new(), false); + let (mut out, mut tag, mut in_tag) = (String::new(), String::new(), false); for c in html.chars() { match c { - '<' => in_tag = true, - '>' => { + '<' => { + in_tag = true; + tag.clear(); + } + '>' if in_tag => { in_tag = false; - out.push(' '); + let name: String = tag + .trim_start() + .trim_start_matches('/') + .chars() + .take_while(|c| c.is_ascii_alphanumeric()) + .collect::() + .to_ascii_lowercase(); + if !INLINE_TAGS.contains(&name.as_str()) { + out.push(' '); + } } - c if !in_tag => out.push(c), - _ => {} + c if in_tag => tag.push(c), + c => out.push(c), } } unescape(&out) @@ -723,6 +744,19 @@ mod tests { assert_eq!(strip_tags("

Hello big world

").split_whitespace().collect::>().join(" "), "Hello big world"); } + /// Issue #105: formatting inside a word keeps the word whole, and blocks, line breaks + /// and cells still keep their words apart. + #[test] + fn stripping_tags_keeps_a_partly_formatted_word_whole() { + let words = |html: &str| strip_tags(html).split_whitespace().collect::>().join(" "); + assert_eq!(words("

format

"), "format"); + assert_eq!(words("

cooperate link

"), "cooperate link"); + assert_eq!(words("

one

two

"), "one two"); + assert_eq!(words("

one
two
three

"), "one two three"); + assert_eq!(words("
  • a
  • b
cd
"), "a b c d"); + assert_eq!(words("

Title

Body & more"), "Title Body & more"); + } + #[test] fn converts_blocks_and_inline_markup() { let md = to_markdown("

Title

One bold and italic and code.

  • a
  • b
"); diff --git a/crates/store/src/search.rs b/crates/store/src/search.rs index 66b450f4..5be4216a 100644 --- a/crates/store/src/search.rs +++ b/crates/store/src/search.rs @@ -107,4 +107,17 @@ mod tests { fn a_missing_root_is_empty_not_an_error() { assert!(search(Path::new("/definitely/not/here"), "x", 5).is_empty()); } + + /// Issue #105: a word that is partly bold on an HTML page is found as the word it reads as. + #[test] + fn a_partly_formatted_word_on_an_html_page_is_found() { + let t = TempDir::new("search-html-inline"); + let s = Settings::at(t.path().join("s.toml")); + let root = t.path().join("nb"); + let nb = ops::create_notebook(&root, "NB", None, &s).expect("nb"); + let page = ops::create_node(&nb, "Plain", NodeKind::Page, Some(PageFormat::Html), &s).expect("page"); + save_body(&page, "

The format of cooperation

").expect("save"); + assert_eq!(search(&root, "format", 10).len(), 1); + assert_eq!(search(&root, "the format of cooperation", 10).len(), 1); + } } From 05143a9ff36e02362a162cfa94d3721d93583149 Mon Sep 17 00:00:00 2001 From: DWAA1660 Date: Thu, 12 Nov 2026 10:24:32 -0700 Subject: [PATCH 07/26] fix(store): search the visible words of CMGFM pages Search read a CMGFM page as its Markdown source, so `Hello world` did not find `Hello **world**`, while the same page in HTML was found (#104). A Markdown page is now read into blocks, written as the HTML the same page would be, and stripped of tags exactly as an HTML page is, so both formats are searched the same way. Fixes #104 Co-Authored-By: Claude Opus 5.5 --- crates/store/src/search.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/store/src/search.rs b/crates/store/src/search.rs index 66b450f4..fea08f25 100644 --- a/crates/store/src/search.rs +++ b/crates/store/src/search.rs @@ -2,7 +2,7 @@ use crate::page::load_page; use crate::tree::{child_dirs, page_file, read_node}; -use notecraft_formats::html; +use notecraft_formats::{boxes, html, render}; use notecraft_model::PageFormat; use std::path::{Path, PathBuf}; @@ -35,9 +35,12 @@ pub fn search(root: &Path, needle: &str, limit: usize) -> Vec { let mut hits = Vec::new(); for dir in pages { let Ok(page) = load_page(&dir) else { continue }; + // The words a reader sees. Markdown is read into blocks and written as the HTML the + // same page would be, so both formats are searched the same way: searched as source, + // a phrase with a bold word in it, `Hello **world**`, was never found (issue #104). let plain = match page.format { PageFormat::Html => html::strip_tags(&page.body), - PageFormat::Cmgfm => page.body.clone(), + PageFormat::Cmgfm => html::strip_tags(&render::blocks_to_html(&boxes::read(PageFormat::Cmgfm, &page.body).0)), }; let plain = plain.split_whitespace().collect::>().join(" "); let title = read_node(&dir).map(|i| i.title).unwrap_or_default(); @@ -85,6 +88,26 @@ mod tests { assert!(search(&root, "absent", 10).is_empty()); } + /// Issue #104: on a CMGFM page a phrase is found whatever marks are round its words, as + /// it is on an HTML page. + #[test] + fn a_phrase_across_formatting_is_found_on_a_markdown_page() { + let t = TempDir::new("search-md-formatting"); + let s = Settings::at(t.path().join("s.toml")); + let root = t.path().join("nb"); + let nb = ops::create_notebook(&root, "NB", None, &s).expect("nb"); + let md = ops::create_node(&nb, "Markdown", NodeKind::Page, Some(PageFormat::Cmgfm), &s).expect("md"); + save_body(&md, "Hello **world**, see the *new* [plan](https://example.test) and `the code`.\n").expect("save"); + let html = ops::create_node(&nb, "Html", NodeKind::Page, Some(PageFormat::Html), &s).expect("html"); + save_body(&html, "

Hello world, see the new plan and the code.

") + .expect("save"); + for phrase in ["Hello world", "see the new plan and the code"] { + let found: Vec = search(&root, phrase, 10).into_iter().map(|hit| hit.title).collect(); + assert!(found.contains(&"Markdown".to_string()) && found.contains(&"Html".to_string()), "{phrase:?} on both pages: {found:?}"); + } + assert!(search(&root, "**world**", 10).is_empty(), "the marks themselves are not words on the page"); + } + #[test] fn limits_and_empty_needles() { let t = TempDir::new("search2"); From b447b06d9450d4c88af88d3c28078af627f18d53 Mon Sep 17 00:00:00 2001 From: DWAA1660 Date: Thu, 12 Nov 2026 10:29:01 -0700 Subject: [PATCH 08/26] fix(formats): keep code that holds a fence line in one block A code block whose text had a line of three backticks was written inside a three-backtick fence, so that line closed the block early and the rest became a paragraph and an empty code block (#106). The HTML to Markdown conversion wrote the same colliding fence. Both writers now fence code with one more backtick than the longest run starting one of its lines, and the parser follows CommonMark: only a run at least as long as the opening fence, with nothing after it, closes it. Fixes #106 Co-Authored-By: Claude Opus 5.5 --- crates/formats/src/html.rs | 29 ++++++++++++++++++++++-- crates/formats/src/markdown.rs | 26 +++++++++++++++++++--- crates/formats/src/render.rs | 40 ++++++++++++++++++++++++++++++++-- 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/crates/formats/src/html.rs b/crates/formats/src/html.rs index 2d5b7455..d7003f6c 100644 --- a/crates/formats/src/html.rs +++ b/crates/formats/src/html.rs @@ -326,6 +326,9 @@ struct Writer { pre_diagram: bool, /// Inside a `
`, which Mermaid's own pages use instead of a `pre`. div_diagram: bool, + /// Where the open code block's fence starts in `out`, so it can be made longer when the + /// code turns out to hold a fence of its own. + fence_at: Option, in_math: bool, in_math_block: bool, span_sizes: Vec, @@ -338,6 +341,26 @@ struct Writer { } impl Writer { + /// Remembers where the fence of a code block about to be written starts. + fn open_fence(&mut self) { + self.fence_at = (!self.in_cell).then_some(self.out.len()); + } + + /// Writes the fence that closes the open code block. When a line of the code starts with + /// three or more backticks, both fences are made longer than that run, so the code stays + /// one block when it is read again (issue #106). + fn close_fence(&mut self) { + let mut fence = "```".to_string(); + if let Some(at) = self.fence_at.take().filter(|&at| self.out.is_char_boundary(at)) { + let code_from = self.out[at..].find('\n').map_or(self.out.len(), |n| at + n + 1); + fence = crate::markdown::code_fence(&self.out[code_from..]); + if fence.len() > 3 { + self.out.insert_str(at, &fence[3..]); + } + } + self.out.push_str(&fence); + } + fn push(&mut self, s: &str) { if self.in_cell { self.cell.push_str(s); @@ -423,6 +446,7 @@ impl Writer { // A div with the class `mermaid` holds a diagram, as a pre does. Its text is the source. ("div", false) if !self.in_pre && attrs.to_lowercase().contains("mermaid") => { self.block_break(); + self.open_fence(); self.push("```mermaid\n"); self.in_pre = true; self.pre_diagram = true; @@ -434,7 +458,7 @@ impl Writer { if !self.out.ends_with('\n') { self.out.push('\n'); } - self.out.push_str("```"); + self.close_fence(); self.block_break(); } ("p", true) if self.in_math_block => { @@ -522,6 +546,7 @@ impl Writer { let class = attr(attrs, "class").unwrap_or_default(); self.pre_diagram = class.eq_ignore_ascii_case("diagram") || class.eq_ignore_ascii_case("mermaid"); let language = if self.pre_diagram { "mermaid" } else { class.as_str() }; + self.open_fence(); self.push(&format!("```{language}\n")); self.in_pre = true; } @@ -530,7 +555,7 @@ impl Writer { if !self.out.ends_with('\n') { self.out.push('\n'); } - self.out.push_str("```"); + self.close_fence(); self.block_break(); } ("span", false) => { diff --git a/crates/formats/src/markdown.rs b/crates/formats/src/markdown.rs index f75dfaa0..4f42a863 100644 --- a/crates/formats/src/markdown.rs +++ b/crates/formats/src/markdown.rs @@ -573,6 +573,18 @@ fn try_parse_img_block(line: &str) -> Option { Some(Block::Image { alt, src, width, height }) } +/// How many backticks `text` starts with. +fn backtick_run(text: &str) -> usize { + text.bytes().take_while(|&b| b == b'`').count() +} + +/// The fence to write round `code`: three backticks, or one more than the longest run that +/// starts a line of the code, so no line of it can close the block early (issue #106). +pub(crate) fn code_fence(code: &str) -> String { + let longest = code.lines().map(|line| backtick_run(line.trim_start())).max().unwrap_or(0); + "`".repeat(longest.max(2) + 1) +} + /// Parses a Markdown page body into blocks. Never panics. pub fn parse(src: &str) -> Vec { let text = src.replace("\r\n", "\n"); @@ -584,11 +596,19 @@ pub fn parse(src: &str) -> Vec { let t = line.trim(); if t.is_empty() { i += 1; - } else if let Some(fence) = t.strip_prefix("```") { - let lang = fence.trim().to_string(); + } else if t.starts_with("```") { + // The fence is as long as its run of backticks, and only a run at least as long, + // with nothing after it, closes it: a shorter fence inside is part of the code. + let open = backtick_run(t); + let lang = t[open..].trim().to_string(); + let closes = |l: &str| { + let l = l.trim(); + let run = backtick_run(l); + run >= open && l[run..].is_empty() + }; let mut body = Vec::new(); i += 1; - while i < lines.len() && !lines.get(i).is_some_and(|l| l.trim_start().starts_with("```")) { + while i < lines.len() && !lines.get(i).is_some_and(|l| closes(l)) { body.push(lines.get(i).copied().unwrap_or("")); i += 1; } diff --git a/crates/formats/src/render.rs b/crates/formats/src/render.rs index 446f9b37..eef6b58b 100644 --- a/crates/formats/src/render.rs +++ b/crates/formats/src/render.rs @@ -48,14 +48,16 @@ pub fn blocks_to_markdown(blocks: &[Block]) -> String { out.push('\n'); } Block::Code { lang, text } => { - out.push_str("```"); + let fence = crate::markdown::code_fence(text); + out.push_str(&fence); out.push_str(lang); out.push('\n'); out.push_str(text); if !text.ends_with('\n') { out.push('\n'); } - out.push_str("```\n"); + out.push_str(&fence); + out.push('\n'); } Block::MathBlock(tex) => { out.push_str("$$\n"); @@ -645,6 +647,40 @@ fn b() {}" } } + /// Issue #106: code that holds a fence line of its own stays one block, in both formats. + /// The fence written round it is longer than any run of backticks starting one of its + /// lines, and only a fence at least as long as the opening one closes the block. + #[test] + fn code_holding_a_fence_stays_one_block() { + let tick = "`"; + for text in [ + format!("before\n{}\nafter", tick.repeat(3)), + format!("{}md\nnested\n{}", tick.repeat(3), tick.repeat(3)), + format!(" {}\n{}python\nx\n{}", tick.repeat(5), tick.repeat(4), tick.repeat(4)), + format!("inline {} only", tick.repeat(3)), + ] { + let original = vec![Block::Code { lang: "markdown".into(), text: text.clone() }]; + let saved = blocks_to_markdown(&original); + assert_eq!(parse(&saved), original, "CMGFM:\n{saved}"); + let html = blocks_to_html(&original); + let reread = html::to_markdown(&html); + assert_eq!(parse(&reread), original, "HTML:\n{html}\n--- read as ---\n{reread}"); + } + // A plain block keeps the three backticks it always had. + assert_eq!( + blocks_to_markdown(&[Block::Code { lang: "rust".into(), text: "let x = 1;".into() }]), + format!("{0}rust\nlet x = 1;\n{0}\n", tick.repeat(3)) + ); + // A shorter fence inside a longer one is code, and a closing fence with words after + // it is not a closing fence. + let four = tick.repeat(4); + let three = tick.repeat(3); + assert_eq!( + parse(&format!("{four}\n{three}\n{three}js\n{four}\n")), + vec![Block::Code { lang: String::new(), text: format!("{three}\n{three}js") }] + ); + } + #[test] fn html_escapes_everything_that_matters() { let blocks = parse("A