diff --git a/book.toml b/book.toml index 0b88649..0ccaf77 100644 --- a/book.toml +++ b/book.toml @@ -8,14 +8,22 @@ create-missing = false [preprocessor.graphviz] command = "mdbook-graphviz" +renderers = ["html"] after = ["links"] [preprocessor.mermaid] command = "python3 contrib/mermaid_ssr.py" after = ["links", "graphviz"] +[preprocessor.pdf] +command = "python3 contrib/pdf_preprocess.py" +renderers = ["pandoc"] +after = ["links", "jupyter"] +before = ["mermaid"] + [preprocessor.katex] command = "mdbook-katex" +renderers = ["html"] after = ["links"] [output.html] diff --git a/contrib/mermaid_ssr.py b/contrib/mermaid_ssr.py index af5283a..3da549b 100644 --- a/contrib/mermaid_ssr.py +++ b/contrib/mermaid_ssr.py @@ -15,6 +15,8 @@ import subprocess import sys import tempfile +import xml.etree.ElementTree as ET +from html import unescape # Fence length varies: preprocessors that re-serialize the markdown (mdbook-graphviz) # emit four backticks, so the closing fence has to match the opening one. @@ -23,6 +25,9 @@ re.S | re.M, ) PLACEHOLDER = "MERMAID_SSR_ID" +SVG = "http://www.w3.org/2000/svg" +ET.register_namespace("", SVG) +ET.register_namespace("xlink", "http://www.w3.org/1999/xlink") def render(source, theme, svg_id): @@ -42,9 +47,16 @@ def render(source, theme, svg_id): svg = os.path.join(tmp, "out.svg") with open(mmd, "w") as f: f.write(source) + args = ["mmdc", "-i", mmd, "-o", svg, + "-b", "transparent", "-t", "default" if theme == "pdf" else theme, + "-I", svg_id] + if theme == "pdf": + config = os.path.join(tmp, "config.json") + with open(config, "w") as f: + json.dump({"htmlLabels": False, "flowchart": {"htmlLabels": False}}, f) + args += ["-c", config] result = subprocess.run( - ["mmdc", "-i", mmd, "-o", svg, - "-b", "transparent", "-t", theme, "-I", svg_id], + args, capture_output=True, text=True, ) @@ -71,11 +83,21 @@ def block(source, index): def image(source, index): - """The light theme of one diagram, as an image for the pandoc renderer. + """One diagram with native SVG labels, as an image for the pandoc renderer. Inline svg is html, which pandoc drops from a LaTeX document. """ - svg = render(source, "default", "mermaid-%d" % index) + svg = render(source, "pdf", "mermaid-%d" % index) + root = ET.fromstring(svg) + # librsvg otherwise collapses leading spaces in adjacent text spans. + root.set("{http://www.w3.org/XML/1998/namespace}space", "preserve") + for label in root.iter("{%s}text" % SVG): + for node in label.iter(): + if node.text: + node.text = unescape(node.text) + if node.tail: + node.tail = unescape(node.tail) + svg = ET.tostring(root, encoding="unicode") return "\n\n![](data:image/svg+xml;base64,%s)\n\n" % base64.b64encode( svg.encode()).decode() diff --git a/contrib/pdf-layout.lua b/contrib/pdf-layout.lua new file mode 100644 index 0000000..bb708f0 --- /dev/null +++ b/contrib/pdf-layout.lua @@ -0,0 +1,75 @@ +local escapes = { + [' ']='\\ ', + ['\\']='\\textbackslash{}', ['{']='\\{', ['}']='\\}', + ['$']='\\$', ['&']='\\&', ['#']='\\#', ['%']='\\%', + ['_']='\\_', ['^']='\\textasciicircum{}', ['~']='\\textasciitilde{}' +} +local function breakable_code(code) + local chars = {} + for _, point in utf8.codes(code.text) do + local char = utf8.char(point) + chars[#chars + 1] = escapes[char] or char + end + return pandoc.RawInline('latex', '\\texttt{' .. table.concat(chars, '\\allowbreak ') .. '}') +end +function Table(tbl) + local headers = {} + if #tbl.head.rows > 0 then + for _, cell in ipairs(tbl.head.rows[1].cells) do + headers[#headers + 1] = pandoc.utils.stringify(cell.contents) + end + end + local key = table.concat(headers, '|') + if key == 'set|value|binary (little endian)|popcount|weight (base 3)|weight (base 10)' + and #tbl.colspecs == 7 then + tbl.head.rows[1].cells:insert(1, pandoc.Cell {}) + key = '|' .. key + end + -- mdbook-pandoc splits these mixed code/text headers into extra cells. + if key == 'Name|keytype|keydata|keydata| Description|valuedata|valuedata| Description' then + local cells = tbl.head.rows[1].cells + for _, i in ipairs {4, 7} do + local label = cells[i].contents[1].content + label:extend(cells[i + 1].contents[1].content) + cells[i].contents = {pandoc.Plain(label)} + end + tbl.head.rows[1].cells = {cells[1], cells[2], cells[3], cells[4], cells[6], cells[7]} + key = 'Name|keytype|keydata|keydata Description|valuedata|valuedata Description' + end + local widths + if key == 'Name|keytype|keydata|keydata Description|valuedata|valuedata Description' then + widths = { .175, .195, .11, .14, .14, .24 } + elseif key == '|PSBT version|BIP' then + widths = { .7, .15, .15 } + elseif key == '|set|value|binary (little endian)|popcount|weight (base 3)|weight (base 10)' then + widths = { .04, .13, .15, .25, .13, .15, .15 } + for _, body in ipairs(tbl.bodies) do + for _, row in ipairs(body.body) do + local cell = row.cells[4] + cell.contents = {pandoc.Plain {pandoc.Code(pandoc.utils.stringify(cell.contents))}} + end + end + end + if widths then + for i, width in ipairs(widths) do tbl.colspecs[i][2] = width end + end + tbl = tbl:walk { Code = breakable_code } + if widths and (#tbl.colspecs == 6 or #tbl.colspecs == 7) then + return {pandoc.RawBlock('latex', '\\begingroup\\small\\setlength{\\tabcolsep}{3pt}'), tbl, pandoc.RawBlock('latex', '\\endgroup')} + end + return tbl +end + +function Para(para) + local images = pandoc.List() + for _, inline in ipairs(para.content) do + if inline.t == 'Image' then + images:insert(pandoc.Para {inline}) + elseif inline.t ~= 'Space' and inline.t ~= 'SoftBreak' then + return nil + end + end + if #images > 1 then return images end +end + +Plain = Para diff --git a/contrib/pdf_preprocess.py b/contrib/pdf_preprocess.py new file mode 100644 index 0000000..0dc94fc --- /dev/null +++ b/contrib/pdf_preprocess.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Expose saved notebook HTML tables to mdbook's Pandoc renderer.""" +import json +import re +import subprocess +import sys + + +TABLE = re.compile(r"]*>.*?", re.I | re.S) + + +def convert_tables(content): + def convert(match): + result = subprocess.run( + ["pandoc", "-f", "html", "-t", "gfm", "--wrap=none"], + input=match.group(), text=True, stdout=subprocess.PIPE, check=True) + # Blank lines end the surrounding Jupyter div's raw HTML block, so + # mdbook parses the replacement as a Markdown table. + return "\n\n" + result.stdout.strip() + "\n\n" + + return TABLE.sub(convert, content) + + +def walk(items): + for item in items: + chapter = item.get("Chapter") if isinstance(item, dict) else None + if chapter is None: + continue + source = chapter.get("source_path") or chapter.get("path") or "" + if source.lower().endswith(".ipynb"): + chapter["content"] = convert_tables(chapter["content"]) + walk(chapter.get("sub_items", [])) + + +def main(): + if len(sys.argv) > 1 and sys.argv[1] == "supports": + sys.exit(0 if sys.argv[2:] == ["pandoc"] else 1) + context, book = json.load(sys.stdin) + if context["renderer"] == "pandoc": + walk(book.get("items", book.get("sections", []))) + json.dump(book, sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/contrib/test_mermaid_ssr.py b/contrib/test_mermaid_ssr.py index 8c7ab0b..45e5807 100644 --- a/contrib/test_mermaid_ssr.py +++ b/contrib/test_mermaid_ssr.py @@ -1,5 +1,6 @@ import base64 import hashlib +import xml.etree.ElementTree as ET import mermaid_ssr import pytest @@ -28,9 +29,16 @@ def test_diagram_missing_from_the_prerendered_set_fails(tmp_path, monkeypatch): mermaid_ssr.render(SOURCE + " b --> c\n", "default", "mermaid-1-light") -def test_pandoc_gets_the_light_diagram_as_an_image(tmp_path, monkeypatch): - prerender(tmp_path, SOURCE, "default", '') +def test_pandoc_uses_pdf_labels_with_spaces_and_decoded_entities(tmp_path, monkeypatch): + prerender(tmp_path, SOURCE, "pdf", ( + '' + "Bob's change 0.1" + ' ? ' + )) monkeypatch.setenv("MERMAID_SVG_DIR", str(tmp_path)) - svg = base64.b64encode(b'').decode() - assert mermaid_ssr.image(SOURCE, 2) == ( - "\n\n![](data:image/svg+xml;base64,%s)\n\n" % svg) + markdown = mermaid_ssr.image(SOURCE, 2) + encoded = markdown.split("base64,", 1)[1].split(")", 1)[0] + svg = ET.fromstring(base64.b64decode(encoded)) + assert svg.attrib["id"] == "mermaid-2" + assert svg.attrib["{http://www.w3.org/XML/1998/namespace}space"] == "preserve" + assert "".join(svg.itertext()) == "Bob's change 0.1\u00a0?\u00a0" diff --git a/contrib/test_pdf_layout.py b/contrib/test_pdf_layout.py new file mode 100644 index 0000000..85ef8a8 --- /dev/null +++ b/contrib/test_pdf_layout.py @@ -0,0 +1,91 @@ +import copy +import json +from pathlib import Path +import subprocess + + +FILTER = Path(__file__).with_name('pdf-layout.lua') + + +def parse_markdown(markdown): + return json.loads(subprocess.run( + ['pandoc', '-f', 'gfm', '-t', 'json'], input=markdown, + text=True, capture_output=True, check=True).stdout) + + +def apply_layout(document): + return json.loads(subprocess.run( + ['pandoc', '-f', 'json', '-t', 'json', '--lua-filter', str(FILTER)], + input=json.dumps(document), text=True, capture_output=True, + check=True).stdout) + + +def only_table(document): + table, = [block['c'] for block in document['blocks'] if block['t'] == 'Table'] + return table + + +def cell_text(cell): + return ''.join(inline['c'] if inline['t'] == 'Str' else ' ' + for block in cell[4] for inline in block['c']) + + +def test_consecutive_notebook_images_each_get_their_own_block(): + document = parse_markdown('![First plot](first.svg) ![Second plot](second.svg)') + block, = document['blocks'] + block['t'] = 'Plain' # mdbook-pandoc emits notebook outputs as Plain. + original = [inline for inline in block['c'] if inline['t'] == 'Image'] + result = apply_layout(document) + assert len(result['blocks']) == 2 + assert all(block['t'] in ['Plain', 'Para'] for block in result['blocks']) + assert [block['c'] for block in result['blocks']] == [[image] for image in original] + + +def test_notebook_index_header_is_restored_without_shifting_column_names(): + document = parse_markdown(''' +| | set | value | binary (little endian) | popcount | weight (base 3) | weight (base 10) | +| --- | --- | --- | --- | --- | --- | --- | +| 0 | ternary | 0.00000486 | 011001111 | 6 | 1 | 3 | +| 1 | decimal | 0.00000500 | 001011111 | 6 | 4 | 1 | +''') + table = only_table(document) + del table[3][1][0][1][0] # mdbook-pandoc omits the empty index header. + result = only_table(apply_layout(document)) + headers = result[3][1][0][1] + assert len(result[2]) == len(headers) == 7 + assert [cell_text(cell) for cell in headers] == [ + '', 'set', 'value', 'binary (little endian)', 'popcount', + 'weight (base 3)', 'weight (base 10)', + ] + rows = result[4][0][3] + assert [len(row[1]) for row in rows] == [7, 7] + assert [[cell_text(cell) for cell in row[1][:3]] for row in rows] == [ + ['0', 'ternary', '0.00000486'], ['1', 'decimal', '0.00000500'], + ] + + +def test_bip_mixed_code_and_text_headers_keep_six_columns(): + document = parse_markdown(''' +| Name | `keytype` | `keydata` | `keydata` Description | `valuedata` | `valuedata` Description | +| --- | --- | --- | --- | --- | --- | +| Input Sort Key | `PSBT_IN_SORT_KEY = TBD` | None | No key data | `` | Orders inputs. | +''') + table = only_table(document) + headers = table[3][1][0][1] + + def split_header(cell): + code, description = copy.deepcopy(cell), copy.deepcopy(cell) + code[4][0]['c'] = code[4][0]['c'][:1] + description[4][0]['c'] = description[4][0]['c'][1:] + return [code, description] + + table[3][1][0][1] = (headers[:3] + split_header(headers[3]) + + [headers[4]] + split_header(headers[5])) + result = only_table(apply_layout(document)) + assert len(result[2]) == len(result[3][1][0][1]) == 6 + assert len(result[4][0][3][0][1]) == 6 + restored = result[3][1][0][1] + for index in [3, 5]: + inlines = restored[index][4][0]['c'] + assert len(inlines) > 1 + assert inlines[-1] == {'t': 'Str', 'c': 'Description'} diff --git a/contrib/test_pdf_preprocess.py b/contrib/test_pdf_preprocess.py new file mode 100644 index 0000000..00ec9d3 --- /dev/null +++ b/contrib/test_pdf_preprocess.py @@ -0,0 +1,69 @@ +import json +from pathlib import Path +import subprocess +import sys + +import pdf_preprocess + + +DATAFRAME = '''
+ + + + + + +
amountlabel
0123Alice & Bob
1456change
+
''' + + +def test_saved_dataframe_becomes_a_markdown_table_inside_its_wrapper(): + content = 'Before output.\n\n' + DATAFRAME + '\n\nAfter output.' + converted = pdf_preprocess.convert_tables(content) + assert converted.startswith('Before output.\n\n
') + assert converted.endswith('
\n\nAfter output.') + assert '} + \fvset{breaklines,breakanywhere,fontsize=\footnotesize} + \usepackage{etoolbox} + % Leave paragraph space around tall images to avoid fvextra's output loop. + \ifdefined\pandocbounded + \patchcmd{\pandocbounded}{\textheight} + {\dimexpr\textheight-\baselineskip-2\parskip\relax} + {}{\errmessage{Cannot reserve image spacing}} + \fi + ''; + }; }; buildPhase = '' export FONTCONFIG_FILE=${fontconfig} + export XDG_CACHE_HOME=$(mktemp -d) python3 contrib/book.py . mdbook build -d book install -D book/pdf/fungi-docs.pdf $out/fungi-docs.pdf @@ -75,7 +99,12 @@ inherit site pdf; book = pkgs.runCommand "book-tests" - { nativeBuildInputs = [ (pkgs.python3.withPackages (ps: [ ps.pytest ps.pyyaml ])) ]; } + { + nativeBuildInputs = [ + pkgs.pandoc + (pkgs.python3.withPackages (ps: [ ps.pytest ps.pyyaml ])) + ]; + } '' cd ${../contrib} pytest -p no:cacheprovider