From a6ac94bb8dfa8347b5b6bec9942e306670f70c23 Mon Sep 17 00:00:00 2001 From: Yuval Kogman Date: Mon, 16 Nov 2026 06:17:18 +0100 Subject: [PATCH] [WIP] Fix PDF math, diagram labels and page layout Use XeLaTeX with native dollar-delimited math and an explicit font for draft markers. Keep KaTeX and Graphviz Markdown rewriting in the HTML pipeline so TeX expressions reach the PDF renderer intact. Render PDF diagrams with native SVG labels, preserving text spacing and decoding label entities. Wrap code and table identifiers, repair split BIP headers, and reserve paragraph space around tall images so the page builder can place them. Convert saved notebook HTML tables before rendering, retain the index column, and use a smaller font for all PDF code listings. Stack consecutive plots so each image is bounded to the page. --- book.toml | 8 +++ contrib/mermaid_ssr.py | 30 +++++++++-- contrib/pdf-layout.lua | 75 ++++++++++++++++++++++++++++ contrib/pdf_preprocess.py | 45 +++++++++++++++++ contrib/test_mermaid_ssr.py | 18 +++++-- contrib/test_pdf_layout.py | 91 ++++++++++++++++++++++++++++++++++ contrib/test_pdf_preprocess.py | 69 ++++++++++++++++++++++++++ cspell/tooling.txt | 29 +++++++++++ nix/mermaid.nix | 6 +++ nix/site.nix | 35 +++++++++++-- 10 files changed, 394 insertions(+), 12 deletions(-) create mode 100644 contrib/pdf-layout.lua create mode 100644 contrib/pdf_preprocess.py create mode 100644 contrib/test_pdf_layout.py create mode 100644 contrib/test_pdf_preprocess.py diff --git a/book.toml b/book.toml index 0b88649..0ccaf77 100644 --- a/book.toml +++ b/book.toml @@ -8,14 +8,22 @@ create-missing = false [preprocessor.graphviz] command = "mdbook-graphviz" +renderers = ["html"] after = ["links"] [preprocessor.mermaid] command = "python3 contrib/mermaid_ssr.py" after = ["links", "graphviz"] +[preprocessor.pdf] +command = "python3 contrib/pdf_preprocess.py" +renderers = ["pandoc"] +after = ["links", "jupyter"] +before = ["mermaid"] + [preprocessor.katex] command = "mdbook-katex" +renderers = ["html"] after = ["links"] [output.html] diff --git a/contrib/mermaid_ssr.py b/contrib/mermaid_ssr.py index af5283a..3da549b 100644 --- a/contrib/mermaid_ssr.py +++ b/contrib/mermaid_ssr.py @@ -15,6 +15,8 @@ import subprocess import sys import tempfile +import xml.etree.ElementTree as ET +from html import unescape # Fence length varies: preprocessors that re-serialize the markdown (mdbook-graphviz) # emit four backticks, so the closing fence has to match the opening one. @@ -23,6 +25,9 @@ re.S | re.M, ) PLACEHOLDER = "MERMAID_SSR_ID" +SVG = "http://www.w3.org/2000/svg" +ET.register_namespace("", SVG) +ET.register_namespace("xlink", "http://www.w3.org/1999/xlink") def render(source, theme, svg_id): @@ -42,9 +47,16 @@ def render(source, theme, svg_id): svg = os.path.join(tmp, "out.svg") with open(mmd, "w") as f: f.write(source) + args = ["mmdc", "-i", mmd, "-o", svg, + "-b", "transparent", "-t", "default" if theme == "pdf" else theme, + "-I", svg_id] + if theme == "pdf": + config = os.path.join(tmp, "config.json") + with open(config, "w") as f: + json.dump({"htmlLabels": False, "flowchart": {"htmlLabels": False}}, f) + args += ["-c", config] result = subprocess.run( - ["mmdc", "-i", mmd, "-o", svg, - "-b", "transparent", "-t", theme, "-I", svg_id], + args, capture_output=True, text=True, ) @@ -71,11 +83,21 @@ def block(source, index): def image(source, index): - """The light theme of one diagram, as an image for the pandoc renderer. + """One diagram with native SVG labels, as an image for the pandoc renderer. Inline svg is html, which pandoc drops from a LaTeX document. """ - svg = render(source, "default", "mermaid-%d" % index) + svg = render(source, "pdf", "mermaid-%d" % index) + root = ET.fromstring(svg) + # librsvg otherwise collapses leading spaces in adjacent text spans. + root.set("{http://www.w3.org/XML/1998/namespace}space", "preserve") + for label in root.iter("{%s}text" % SVG): + for node in label.iter(): + if node.text: + node.text = unescape(node.text) + if node.tail: + node.tail = unescape(node.tail) + svg = ET.tostring(root, encoding="unicode") return "\n\n![](data:image/svg+xml;base64,%s)\n\n" % base64.b64encode( svg.encode()).decode() diff --git a/contrib/pdf-layout.lua b/contrib/pdf-layout.lua new file mode 100644 index 0000000..bb708f0 --- /dev/null +++ b/contrib/pdf-layout.lua @@ -0,0 +1,75 @@ +local escapes = { + [' ']='\\ ', + ['\\']='\\textbackslash{}', ['{']='\\{', ['}']='\\}', + ['$']='\\$', ['&']='\\&', ['#']='\\#', ['%']='\\%', + ['_']='\\_', ['^']='\\textasciicircum{}', ['~']='\\textasciitilde{}' +} +local function breakable_code(code) + local chars = {} + for _, point in utf8.codes(code.text) do + local char = utf8.char(point) + chars[#chars + 1] = escapes[char] or char + end + return pandoc.RawInline('latex', '\\texttt{' .. table.concat(chars, '\\allowbreak ') .. '}') +end +function Table(tbl) + local headers = {} + if #tbl.head.rows > 0 then + for _, cell in ipairs(tbl.head.rows[1].cells) do + headers[#headers + 1] = pandoc.utils.stringify(cell.contents) + end + end + local key = table.concat(headers, '|') + if key == 'set|value|binary (little endian)|popcount|weight (base 3)|weight (base 10)' + and #tbl.colspecs == 7 then + tbl.head.rows[1].cells:insert(1, pandoc.Cell {}) + key = '|' .. key + end + -- mdbook-pandoc splits these mixed code/text headers into extra cells. + if key == 'Name|keytype|keydata|keydata| Description|valuedata|valuedata| Description' then + local cells = tbl.head.rows[1].cells + for _, i in ipairs {4, 7} do + local label = cells[i].contents[1].content + label:extend(cells[i + 1].contents[1].content) + cells[i].contents = {pandoc.Plain(label)} + end + tbl.head.rows[1].cells = {cells[1], cells[2], cells[3], cells[4], cells[6], cells[7]} + key = 'Name|keytype|keydata|keydata Description|valuedata|valuedata Description' + end + local widths + if key == 'Name|keytype|keydata|keydata Description|valuedata|valuedata Description' then + widths = { .175, .195, .11, .14, .14, .24 } + elseif key == '|PSBT version|BIP' then + widths = { .7, .15, .15 } + elseif key == '|set|value|binary (little endian)|popcount|weight (base 3)|weight (base 10)' then + widths = { .04, .13, .15, .25, .13, .15, .15 } + for _, body in ipairs(tbl.bodies) do + for _, row in ipairs(body.body) do + local cell = row.cells[4] + cell.contents = {pandoc.Plain {pandoc.Code(pandoc.utils.stringify(cell.contents))}} + end + end + end + if widths then + for i, width in ipairs(widths) do tbl.colspecs[i][2] = width end + end + tbl = tbl:walk { Code = breakable_code } + if widths and (#tbl.colspecs == 6 or #tbl.colspecs == 7) then + return {pandoc.RawBlock('latex', '\\begingroup\\small\\setlength{\\tabcolsep}{3pt}'), tbl, pandoc.RawBlock('latex', '\\endgroup')} + end + return tbl +end + +function Para(para) + local images = pandoc.List() + for _, inline in ipairs(para.content) do + if inline.t == 'Image' then + images:insert(pandoc.Para {inline}) + elseif inline.t ~= 'Space' and inline.t ~= 'SoftBreak' then + return nil + end + end + if #images > 1 then return images end +end + +Plain = Para diff --git a/contrib/pdf_preprocess.py b/contrib/pdf_preprocess.py new file mode 100644 index 0000000..0dc94fc --- /dev/null +++ b/contrib/pdf_preprocess.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Expose saved notebook HTML tables to mdbook's Pandoc renderer.""" +import json +import re +import subprocess +import sys + + +TABLE = re.compile(r"]*>.*?", re.I | re.S) + + +def convert_tables(content): + def convert(match): + result = subprocess.run( + ["pandoc", "-f", "html", "-t", "gfm", "--wrap=none"], + input=match.group(), text=True, stdout=subprocess.PIPE, check=True) + # Blank lines end the surrounding Jupyter div's raw HTML block, so + # mdbook parses the replacement as a Markdown table. + return "\n\n" + result.stdout.strip() + "\n\n" + + return TABLE.sub(convert, content) + + +def walk(items): + for item in items: + chapter = item.get("Chapter") if isinstance(item, dict) else None + if chapter is None: + continue + source = chapter.get("source_path") or chapter.get("path") or "" + if source.lower().endswith(".ipynb"): + chapter["content"] = convert_tables(chapter["content"]) + walk(chapter.get("sub_items", [])) + + +def main(): + if len(sys.argv) > 1 and sys.argv[1] == "supports": + sys.exit(0 if sys.argv[2:] == ["pandoc"] else 1) + context, book = json.load(sys.stdin) + if context["renderer"] == "pandoc": + walk(book.get("items", book.get("sections", []))) + json.dump(book, sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/contrib/test_mermaid_ssr.py b/contrib/test_mermaid_ssr.py index 8c7ab0b..45e5807 100644 --- a/contrib/test_mermaid_ssr.py +++ b/contrib/test_mermaid_ssr.py @@ -1,5 +1,6 @@ import base64 import hashlib +import xml.etree.ElementTree as ET import mermaid_ssr import pytest @@ -28,9 +29,16 @@ def test_diagram_missing_from_the_prerendered_set_fails(tmp_path, monkeypatch): mermaid_ssr.render(SOURCE + " b --> c\n", "default", "mermaid-1-light") -def test_pandoc_gets_the_light_diagram_as_an_image(tmp_path, monkeypatch): - prerender(tmp_path, SOURCE, "default", '') +def test_pandoc_uses_pdf_labels_with_spaces_and_decoded_entities(tmp_path, monkeypatch): + prerender(tmp_path, SOURCE, "pdf", ( + '' + "Bob's change 0.1" + ' ? ' + )) monkeypatch.setenv("MERMAID_SVG_DIR", str(tmp_path)) - svg = base64.b64encode(b'').decode() - assert mermaid_ssr.image(SOURCE, 2) == ( - "\n\n![](data:image/svg+xml;base64,%s)\n\n" % svg) + markdown = mermaid_ssr.image(SOURCE, 2) + encoded = markdown.split("base64,", 1)[1].split(")", 1)[0] + svg = ET.fromstring(base64.b64decode(encoded)) + assert svg.attrib["id"] == "mermaid-2" + assert svg.attrib["{http://www.w3.org/XML/1998/namespace}space"] == "preserve" + assert "".join(svg.itertext()) == "Bob's change 0.1\u00a0?\u00a0" diff --git a/contrib/test_pdf_layout.py b/contrib/test_pdf_layout.py new file mode 100644 index 0000000..85ef8a8 --- /dev/null +++ b/contrib/test_pdf_layout.py @@ -0,0 +1,91 @@ +import copy +import json +from pathlib import Path +import subprocess + + +FILTER = Path(__file__).with_name('pdf-layout.lua') + + +def parse_markdown(markdown): + return json.loads(subprocess.run( + ['pandoc', '-f', 'gfm', '-t', 'json'], input=markdown, + text=True, capture_output=True, check=True).stdout) + + +def apply_layout(document): + return json.loads(subprocess.run( + ['pandoc', '-f', 'json', '-t', 'json', '--lua-filter', str(FILTER)], + input=json.dumps(document), text=True, capture_output=True, + check=True).stdout) + + +def only_table(document): + table, = [block['c'] for block in document['blocks'] if block['t'] == 'Table'] + return table + + +def cell_text(cell): + return ''.join(inline['c'] if inline['t'] == 'Str' else ' ' + for block in cell[4] for inline in block['c']) + + +def test_consecutive_notebook_images_each_get_their_own_block(): + document = parse_markdown('![First plot](first.svg) ![Second plot](second.svg)') + block, = document['blocks'] + block['t'] = 'Plain' # mdbook-pandoc emits notebook outputs as Plain. + original = [inline for inline in block['c'] if inline['t'] == 'Image'] + result = apply_layout(document) + assert len(result['blocks']) == 2 + assert all(block['t'] in ['Plain', 'Para'] for block in result['blocks']) + assert [block['c'] for block in result['blocks']] == [[image] for image in original] + + +def test_notebook_index_header_is_restored_without_shifting_column_names(): + document = parse_markdown(''' +| | set | value | binary (little endian) | popcount | weight (base 3) | weight (base 10) | +| --- | --- | --- | --- | --- | --- | --- | +| 0 | ternary | 0.00000486 | 011001111 | 6 | 1 | 3 | +| 1 | decimal | 0.00000500 | 001011111 | 6 | 4 | 1 | +''') + table = only_table(document) + del table[3][1][0][1][0] # mdbook-pandoc omits the empty index header. + result = only_table(apply_layout(document)) + headers = result[3][1][0][1] + assert len(result[2]) == len(headers) == 7 + assert [cell_text(cell) for cell in headers] == [ + '', 'set', 'value', 'binary (little endian)', 'popcount', + 'weight (base 3)', 'weight (base 10)', + ] + rows = result[4][0][3] + assert [len(row[1]) for row in rows] == [7, 7] + assert [[cell_text(cell) for cell in row[1][:3]] for row in rows] == [ + ['0', 'ternary', '0.00000486'], ['1', 'decimal', '0.00000500'], + ] + + +def test_bip_mixed_code_and_text_headers_keep_six_columns(): + document = parse_markdown(''' +| Name | `keytype` | `keydata` | `keydata` Description | `valuedata` | `valuedata` Description | +| --- | --- | --- | --- | --- | --- | +| Input Sort Key | `PSBT_IN_SORT_KEY = TBD` | None | No key data | `` | Orders inputs. | +''') + table = only_table(document) + headers = table[3][1][0][1] + + def split_header(cell): + code, description = copy.deepcopy(cell), copy.deepcopy(cell) + code[4][0]['c'] = code[4][0]['c'][:1] + description[4][0]['c'] = description[4][0]['c'][1:] + return [code, description] + + table[3][1][0][1] = (headers[:3] + split_header(headers[3]) + + [headers[4]] + split_header(headers[5])) + result = only_table(apply_layout(document)) + assert len(result[2]) == len(result[3][1][0][1]) == 6 + assert len(result[4][0][3][0][1]) == 6 + restored = result[3][1][0][1] + for index in [3, 5]: + inlines = restored[index][4][0]['c'] + assert len(inlines) > 1 + assert inlines[-1] == {'t': 'Str', 'c': 'Description'} diff --git a/contrib/test_pdf_preprocess.py b/contrib/test_pdf_preprocess.py new file mode 100644 index 0000000..00ec9d3 --- /dev/null +++ b/contrib/test_pdf_preprocess.py @@ -0,0 +1,69 @@ +import json +from pathlib import Path +import subprocess +import sys + +import pdf_preprocess + + +DATAFRAME = '''
+ + + + + + +
amountlabel
0123Alice & Bob
1456change
+
''' + + +def test_saved_dataframe_becomes_a_markdown_table_inside_its_wrapper(): + content = 'Before output.\n\n' + DATAFRAME + '\n\nAfter output.' + converted = pdf_preprocess.convert_tables(content) + assert converted.startswith('Before output.\n\n
') + assert converted.endswith('
\n\nAfter output.') + assert '} + \fvset{breaklines,breakanywhere,fontsize=\footnotesize} + \usepackage{etoolbox} + % Leave paragraph space around tall images to avoid fvextra's output loop. + \ifdefined\pandocbounded + \patchcmd{\pandocbounded}{\textheight} + {\dimexpr\textheight-\baselineskip-2\parskip\relax} + {}{\errmessage{Cannot reserve image spacing}} + \fi + ''; + }; }; buildPhase = '' export FONTCONFIG_FILE=${fontconfig} + export XDG_CACHE_HOME=$(mktemp -d) python3 contrib/book.py . mdbook build -d book install -D book/pdf/fungi-docs.pdf $out/fungi-docs.pdf @@ -75,7 +99,12 @@ inherit site pdf; book = pkgs.runCommand "book-tests" - { nativeBuildInputs = [ (pkgs.python3.withPackages (ps: [ ps.pytest ps.pyyaml ])) ]; } + { + nativeBuildInputs = [ + pkgs.pandoc + (pkgs.python3.withPackages (ps: [ ps.pytest ps.pyyaml ])) + ]; + } '' cd ${../contrib} pytest -p no:cacheprovider