pulldown-cmark-to-cmark 23

CHANGELOG.md

@@ -1,3 +1,12 @@
+## [Unreleased]
+
+- Breaking: Markdown output (`to_markdown`, `chunks_by_heading`, `chunks_by_size`, `truncate_markdown`) uses pulldown-cmark-to-cmark 23; escaping and formatting differ from 0.2.
+- Fix raw HTML leaking into Markdown output with `raw_html: false` when a line held more than one tag.
+- Fix `chunks_by_size` and `truncate_markdown` (`at: :block`) joining blocks without a blank line, which merged adjacent paragraphs.
+- Fix Markdown output crashing on an image nested in another image's alt text with `images: { lazy: true }`, `allowed_hosts`, or `allowed_schemes`.
+- `chunks_by_heading` starts sections only at top-level headings; a heading inside a block quote or list item no longer splits it.
+- Raise `Inkmark::Error` on Markdown serialization failures instead of crashing the process.
+
 ## [0.2.2] - 2026-10-24
 
 - Updated dependencies, including pulldown-cmark 0.13.5 (upstream security fixes) and magnus 0.9.3.

README.md

@@ -206,6 +206,9 @@ Inkmark.to_html("<script>alert(1)</script>")
 # => "<p>&lt;script&gt;alert(1)&lt;/script&gt;</p>\n"
 ```
 
+Markdown output escapes it too; see
+[Raw HTML in the source](#raw-html-in-the-source).
+
 Enable pass-through with `raw_html: true`; _only do this for trusted
 input_:
 
@@ -403,6 +406,12 @@ nested `###` subsections, which also appear as their own entries. Content
 before the first heading (if any) becomes a preamble entry with
 `heading: nil` and `level: 0`.
 
+Only **top-level** headings start a section. A heading inside a block
+quote or list item (such as a `## Tip` in a `> [!NOTE]` callout) stays in
+the content of the section around it, so the quote or list is never split
+across sections. It still counts toward slug deduplication, so every
+section's `:id` matches the anchor `headings: { ids: true }` generates.
+
 `:breadcrumb` carries the ancestor heading texts from root to immediate
 parent. Root-level sections and the preamble have an empty array. Skipped
 levels are omitted, so an `###` directly under an `#` has `breadcrumb:
@@ -740,6 +749,17 @@ Inkmark.to_markdown(source, options: {
 })
 ```
 
+### Raw HTML in the source
+
+With `raw_html: false` (the default), raw HTML from the source comes out of
+`to_markdown`, `chunks_by_heading`, `chunks_by_size`, and `truncate_markdown`
+escaped, so the Markdown stays inert even if a later stage renders it with raw
+HTML allowed. HTML produced by the HTML-emitting filters above is the
+exception: it is embedded as described.
+
+If the filtered document cannot be written back as Markdown, these methods
+raise `Inkmark::Error`.
+
 ## Event handlers
 
 Register handlers with `#on` to inspect or transform document elements as they

ext/inkmark/Cargo.toml

@@ -22,7 +22,7 @@ syntect = { version = "5.2", default-features = false, features = ["parsing", "h
 linkify = "0.11"
 globset = "0.4"
 url = "2"
-pulldown-cmark-to-cmark = "22"
+pulldown-cmark-to-cmark = "23"
 
 [build-dependencies]
 rb-sys-env = "0.2.2"

ext/inkmark/src/chunks_by_heading.rs

@@ -14,13 +14,20 @@
 //! emoji/autolink rewrites happen *inside* a heading's text events,
 //! but the bracketing tags stay in place—so post-filter heading-
 //! position scanning is coherent.
+//!
+//! Only top-level headings start a section. A heading inside a block
+//! quote, list item, or other container stays in the content of the
+//! section around it, so a container is never split across sections.
+//! Nested headings still consume slugs, keeping section ids aligned
+//! with the ids `heading_ids` / `toc` emit.
 
 use magnus::{Error, RArray, RHash, Ruby};
 use pulldown_cmark::{Event, HeadingLevel, Parser, Tag, TagEnd};
 use unicode_segmentation::UnicodeSegmentation;
 
 use crate::document::{apply_filters, content_events};
 use crate::heading::{self, SlugDeduplicator};
+use crate::markdown;
 use crate::options::build_options;
 use crate::toc;
 use crate::truncate::{self, TruncateParams};
@@ -57,8 +64,13 @@ pub fn native_chunks_by_heading(
     let content_opts = ContentOptions {
         with_counts: flags.statistics,
         truncate_params: truncate_params.as_ref(),
+        md_opts: markdown::serializer_options(cm_opts, &flags),
     };
-    let preamble_end = boundaries.first().map(|b| b.start).unwrap_or(events.len());
+    let preamble_end = boundaries
+        .iter()
+        .find(|b| b.top_level)
+        .map(|b| b.start)
+        .unwrap_or(events.len());
     if preamble_end > 0 {
         let preamble_events = &events[0..preamble_end];
         if !is_empty_content(preamble_events) {
@@ -78,23 +90,29 @@ pub fn native_chunks_by_heading(
     let mut dedup = SlugDeduplicator::new();
     let mut ancestors: Vec<(u8, String)> = Vec::new();
     for (i, boundary) in boundaries.iter().enumerate() {
+        let heading_text = collect_inline_text(&events[(boundary.start + 1)..boundary.end]);
+        let id = heading_id(&mut dedup, &heading_text);
+        if !boundary.top_level {
+            // Stays in the surrounding section's content; only its slug
+            // counts, so later ids match `heading_ids`.
+            continue;
+        }
         let section_end = find_section_end(&boundaries, i, events.len());
         let level = toc::level_to_u8(boundary.level);
         while ancestors.last().is_some_and(|(l, _)| *l >= level) {
             ancestors.pop();
         }
         let breadcrumb: Vec<&str> = ancestors.iter().map(|(_, t)| t.as_str()).collect();
-        let heading_text = collect_inline_text(&events[(boundary.start + 1)..boundary.end]);
         // Content = events after End(Heading) up to the next section or
-        // end of document. Re-serialized through cmark_write.
+        // end of document. Re-serialized through `markdown::render`.
         let content_events = &events[(boundary.end + 1)..section_end];
         let hash = build_section_hash(
             ruby,
             boundary.level,
             &heading_text,
+            &id,
             &breadcrumb,
             content_events,
-            &mut dedup,
             &content_opts,
         )?;
         result.push(hash)?;
@@ -110,21 +128,30 @@ struct HeadingBoundary {
     start: usize,
     end: usize,
     level: HeadingLevel,
+    /// Not inside a block quote, list item, or other container. Only
+    /// top-level headings start sections.
+    top_level: bool,
 }
 
 /// How each entry's `content` is rendered: optionally truncated, and
 /// optionally accompanied by `character_count` / `word_count`.
 struct ContentOptions<'a> {
     with_counts: bool,
     truncate_params: Option<&'a TruncateParams>,
+    /// Serializer options, from `markdown::serializer_options`.
+    md_opts: pulldown_cmark::Options,
 }
 
 fn find_heading_boundaries(events: &[Event<'_>]) -> Vec<HeadingBoundary> {
     let mut boundaries = Vec::new();
+    // Containers open around the cursor. Headings are leaf blocks, so
+    // none open at a Start(Heading) means it is not inside anything.
+    let mut open_containers = 0usize;
     let mut i = 0;
     while i < events.len() {
         if let Event::Start(Tag::Heading { level, .. }) = &events[i] {
             let lvl = *level;
+            let top_level = open_containers == 0;
             // CommonMark disallows headings inside headings, but carry a
             // depth counter so we stay correct if pulldown-cmark ever
             // permits them.
@@ -147,23 +174,29 @@ fn find_heading_boundaries(events: &[Event<'_>]) -> Vec<HeadingBoundary> {
                 start: i,
                 end: j,
                 level: lvl,
+                top_level,
             });
             i = j + 1;
         } else {
+            match &events[i] {
+                Event::Start(_) => open_containers += 1,
+                Event::End(_) => open_containers = open_containers.saturating_sub(1),
+                _ => {}
+            }
             i += 1;
         }
     }
     boundaries
 }
 
-/// Section i ends at the first subsequent heading with level <=
-/// current. Headings with a strictly greater level are subsections:
+/// Section i ends at the first subsequent top-level heading with level
+/// <= current. Headings with a strictly greater level are subsections:
 /// they belong to the current section's content too.
 fn find_section_end(boundaries: &[HeadingBoundary], i: usize, events_len: usize) -> usize {
     let current = toc::level_to_u8(boundaries[i].level);
     boundaries[(i + 1)..]
         .iter()
-        .find(|b| toc::level_to_u8(b.level) <= current)
+        .find(|b| b.top_level && toc::level_to_u8(b.level) <= current)
         .map(|b| b.start)
         .unwrap_or(events_len)
 }
@@ -181,10 +214,7 @@ fn build_preamble_hash(
     // with proper sections so callers can treat every entry alike.
     hash.aset(ruby.to_symbol("breadcrumb"), ruby.ary_new_capa(0))?;
 
-    let content = match content_opts.truncate_params {
-        Some(params) => truncate::truncate_events(events, params),
-        None => render_markdown(events),
-    };
+    let content = render_content(ruby, events, content_opts)?;
     if content_opts.with_counts {
         let (chars, words) = count_post_truncate(events, content_opts.truncate_params, &content);
         hash.aset(ruby.to_symbol("character_count"), chars)?;
@@ -198,21 +228,11 @@ fn build_section_hash(
     ruby: &Ruby,
     level: HeadingLevel,
     heading_text: &str,
+    id: &str,
     breadcrumb: &[&str],
     content_events: &[Event<'_>],
-    dedup: &mut SlugDeduplicator,
     content_opts: &ContentOptions<'_>,
 ) -> Result<RHash, Error> {
-    // Slug is the deduplicated slugify of the (filter-applied) heading
-    // text, matching the ids `heading_ids` / `toc` would emit for the
-    // same document.
-    let base = heading::slugify(heading_text);
-    let id = if base.is_empty() {
-        String::new()
-    } else {
-        dedup.deduplicate(base)
-    };
-
     let hash = ruby.hash_new();
     hash.aset(ruby.to_symbol("heading"), heading_text)?;
     hash.aset(ruby.to_symbol("level"), toc::level_to_u8(level))?;
@@ -227,10 +247,7 @@ fn build_section_hash(
     }
     hash.aset(ruby.to_symbol("breadcrumb"), breadcrumb_arr)?;
 
-    let content = match content_opts.truncate_params {
-        Some(params) => truncate::truncate_events(content_events, params),
-        None => render_markdown(content_events),
-    };
+    let content = render_content(ruby, content_events, content_opts)?;
     if content_opts.with_counts {
         let (chars, words) =
             count_post_truncate(content_events, content_opts.truncate_params, &content);
@@ -289,11 +306,24 @@ fn count_text(events: &[Event<'_>]) -> (usize, usize) {
     (chars, words)
 }
 
-fn render_markdown(events: &[Event<'_>]) -> String {
-    let mut buf = String::new();
-    pulldown_cmark_to_cmark::cmark(events.iter().cloned(), &mut buf)
-        .expect("markdown serialization failed");
-    buf
+/// Deduplicated slug of the (filter-applied) heading text, matching the
+/// ids `heading_ids` / `toc` would emit for the same document. Empty when
+/// the text has nothing to slugify.
+fn heading_id(dedup: &mut SlugDeduplicator, heading_text: &str) -> String {
+    dedup.deduplicate(heading::slugify(heading_text))
+}
+
+/// Serialize a section's events to Markdown, truncated when requested.
+fn render_content(
+    ruby: &Ruby,
+    events: &[Event<'_>],
+    content_opts: &ContentOptions<'_>,
+) -> Result<String, Error> {
+    match content_opts.truncate_params {
+        Some(params) => truncate::truncate_events(events, params, content_opts.md_opts),
+        None => markdown::render(events, content_opts.md_opts),
+    }
+    .map_err(|e| markdown::to_ruby_error(ruby, e))
 }
 
 fn collect_inline_text(events: &[Event<'_>]) -> String {
@@ -324,3 +354,33 @@ fn is_empty_content(events: &[Event<'_>]) -> bool {
         )
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pulldown_cmark::Options;
+
+    fn events(source: &str) -> Vec<Event<'_>> {
+        Parser::new_ext(source, Options::empty()).collect()
+    }
+
+    #[test]
+    fn headings_inside_containers_are_not_top_level() {
+        let ev = events("# Top\n\n> ## Quoted\n\n- # Listed\n\n## Sub\n");
+        let top_level: Vec<bool> = find_heading_boundaries(&ev)
+            .iter()
+            .map(|b| b.top_level)
+            .collect();
+        assert_eq!(top_level, vec![true, false, false, true]);
+    }
+
+    #[test]
+    fn section_spanning_a_nested_heading_stays_balanced() {
+        let ev = events("# Top\n\n> ## Inner\n> text\n\nafter\n");
+        let boundaries = find_heading_boundaries(&ev);
+        let end = find_section_end(&boundaries, 0, ev.len());
+        let content = &ev[(boundaries[0].end + 1)..end];
+        let md = markdown::render(content, Options::empty()).expect("balanced slice");
+        assert!(md.contains("Inner") && md.contains("after"), "{md:?}");
+    }
+}