Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions book.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,22 @@ create-missing = false

[preprocessor.graphviz]
command = "mdbook-graphviz"
renderers = ["html"]
after = ["links"]

[preprocessor.mermaid]
command = "python3 contrib/mermaid_ssr.py"
after = ["links", "graphviz"]

[preprocessor.pdf]
command = "python3 contrib/pdf_preprocess.py"
renderers = ["pandoc"]
after = ["links", "jupyter"]
before = ["mermaid"]

[preprocessor.katex]
command = "mdbook-katex"
renderers = ["html"]
after = ["links"]

[output.html]
Expand Down
30 changes: 26 additions & 4 deletions contrib/mermaid_ssr.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
import subprocess
import sys
import tempfile
import xml.etree.ElementTree as ET
from html import unescape

# Fence length varies: preprocessors that re-serialize the markdown (mdbook-graphviz)
# emit four backticks, so the closing fence has to match the opening one.
Expand All @@ -23,6 +25,9 @@
re.S | re.M,
)
PLACEHOLDER = "MERMAID_SSR_ID"
SVG = "http://www.w3.org/2000/svg"
ET.register_namespace("", SVG)
ET.register_namespace("xlink", "http://www.w3.org/1999/xlink")


def render(source, theme, svg_id):
Expand All @@ -42,9 +47,16 @@ def render(source, theme, svg_id):
svg = os.path.join(tmp, "out.svg")
with open(mmd, "w") as f:
f.write(source)
args = ["mmdc", "-i", mmd, "-o", svg,
"-b", "transparent", "-t", "default" if theme == "pdf" else theme,
"-I", svg_id]
if theme == "pdf":
config = os.path.join(tmp, "config.json")
with open(config, "w") as f:
json.dump({"htmlLabels": False, "flowchart": {"htmlLabels": False}}, f)
args += ["-c", config]
result = subprocess.run(
["mmdc", "-i", mmd, "-o", svg,
"-b", "transparent", "-t", theme, "-I", svg_id],
args,
capture_output=True,
text=True,
)
Expand All @@ -71,11 +83,21 @@ def block(source, index):


def image(source, index):
"""The light theme of one diagram, as an image for the pandoc renderer.
"""One diagram with native SVG labels, as an image for the pandoc renderer.

Inline svg is html, which pandoc drops from a LaTeX document.
"""
svg = render(source, "default", "mermaid-%d" % index)
svg = render(source, "pdf", "mermaid-%d" % index)
root = ET.fromstring(svg)
# librsvg otherwise collapses leading spaces in adjacent text spans.
root.set("{http://www.w3.org/XML/1998/namespace}space", "preserve")
for label in root.iter("{%s}text" % SVG):
for node in label.iter():
if node.text:
node.text = unescape(node.text)
if node.tail:
node.tail = unescape(node.tail)
svg = ET.tostring(root, encoding="unicode")
return "\n\n![](data:image/svg+xml;base64,%s)\n\n" % base64.b64encode(
svg.encode()).decode()

Expand Down
75 changes: 75 additions & 0 deletions contrib/pdf-layout.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
local escapes = {
[' ']='\\ ',
['\\']='\\textbackslash{}', ['{']='\\{', ['}']='\\}',
['$']='\\$', ['&']='\\&', ['#']='\\#', ['%']='\\%',
['_']='\\_', ['^']='\\textasciicircum{}', ['~']='\\textasciitilde{}'
}
local function breakable_code(code)
local chars = {}
for _, point in utf8.codes(code.text) do
local char = utf8.char(point)
chars[#chars + 1] = escapes[char] or char
end
return pandoc.RawInline('latex', '\\texttt{' .. table.concat(chars, '\\allowbreak ') .. '}')
end
function Table(tbl)
local headers = {}
if #tbl.head.rows > 0 then
for _, cell in ipairs(tbl.head.rows[1].cells) do
headers[#headers + 1] = pandoc.utils.stringify(cell.contents)
end
end
local key = table.concat(headers, '|')
if key == 'set|value|binary (little endian)|popcount|weight (base 3)|weight (base 10)'
and #tbl.colspecs == 7 then
tbl.head.rows[1].cells:insert(1, pandoc.Cell {})
key = '|' .. key
end
-- mdbook-pandoc splits these mixed code/text headers into extra cells.
if key == 'Name|keytype|keydata|keydata| Description|valuedata|valuedata| Description' then
local cells = tbl.head.rows[1].cells
for _, i in ipairs {4, 7} do
local label = cells[i].contents[1].content
label:extend(cells[i + 1].contents[1].content)
cells[i].contents = {pandoc.Plain(label)}
end
tbl.head.rows[1].cells = {cells[1], cells[2], cells[3], cells[4], cells[6], cells[7]}
key = 'Name|keytype|keydata|keydata Description|valuedata|valuedata Description'
end
local widths
if key == 'Name|keytype|keydata|keydata Description|valuedata|valuedata Description' then
widths = { .175, .195, .11, .14, .14, .24 }
elseif key == '<keytype>|PSBT version|BIP' then
widths = { .7, .15, .15 }
elseif key == '|set|value|binary (little endian)|popcount|weight (base 3)|weight (base 10)' then
widths = { .04, .13, .15, .25, .13, .15, .15 }
for _, body in ipairs(tbl.bodies) do
for _, row in ipairs(body.body) do
local cell = row.cells[4]
cell.contents = {pandoc.Plain {pandoc.Code(pandoc.utils.stringify(cell.contents))}}
end
end
end
if widths then
for i, width in ipairs(widths) do tbl.colspecs[i][2] = width end
end
tbl = tbl:walk { Code = breakable_code }
if widths and (#tbl.colspecs == 6 or #tbl.colspecs == 7) then
return {pandoc.RawBlock('latex', '\\begingroup\\small\\setlength{\\tabcolsep}{3pt}'), tbl, pandoc.RawBlock('latex', '\\endgroup')}
end
return tbl
end

function Para(para)
local images = pandoc.List()
for _, inline in ipairs(para.content) do
if inline.t == 'Image' then
images:insert(pandoc.Para {inline})
elseif inline.t ~= 'Space' and inline.t ~= 'SoftBreak' then
return nil
end
end
if #images > 1 then return images end
end

Plain = Para
45 changes: 45 additions & 0 deletions contrib/pdf_preprocess.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Expose saved notebook HTML tables to mdbook's Pandoc renderer."""
import json
import re
import subprocess
import sys


TABLE = re.compile(r"<table\b[^>]*>.*?</table\s*>", re.I | re.S)


def convert_tables(content):
def convert(match):
result = subprocess.run(
["pandoc", "-f", "html", "-t", "gfm", "--wrap=none"],
input=match.group(), text=True, stdout=subprocess.PIPE, check=True)
# Blank lines end the surrounding Jupyter div's raw HTML block, so
# mdbook parses the replacement as a Markdown table.
return "\n\n" + result.stdout.strip() + "\n\n"

return TABLE.sub(convert, content)


def walk(items):
for item in items:
chapter = item.get("Chapter") if isinstance(item, dict) else None
if chapter is None:
continue
source = chapter.get("source_path") or chapter.get("path") or ""
if source.lower().endswith(".ipynb"):
chapter["content"] = convert_tables(chapter["content"])
walk(chapter.get("sub_items", []))


def main():
if len(sys.argv) > 1 and sys.argv[1] == "supports":
sys.exit(0 if sys.argv[2:] == ["pandoc"] else 1)
context, book = json.load(sys.stdin)
if context["renderer"] == "pandoc":
walk(book.get("items", book.get("sections", [])))
json.dump(book, sys.stdout)


if __name__ == "__main__":
main()
18 changes: 13 additions & 5 deletions contrib/test_mermaid_ssr.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import base64
import hashlib
import xml.etree.ElementTree as ET

import mermaid_ssr
import pytest
Expand Down Expand Up @@ -28,9 +29,16 @@ def test_diagram_missing_from_the_prerendered_set_fails(tmp_path, monkeypatch):
mermaid_ssr.render(SOURCE + " b --> c\n", "default", "mermaid-1-light")


def test_pandoc_gets_the_light_diagram_as_an_image(tmp_path, monkeypatch):
prerender(tmp_path, SOURCE, "default", '<svg id="MERMAID_SSR_ID"/>')
def test_pandoc_uses_pdf_labels_with_spaces_and_decoded_entities(tmp_path, monkeypatch):
prerender(tmp_path, SOURCE, "pdf", (
'<svg xmlns="http://www.w3.org/2000/svg" id="MERMAID_SSR_ID"><text>'
"<tspan>Bob's</tspan><tspan> change</tspan><tspan> 0.1</tspan>"
'<tspan>&amp;nbsp;?&amp;nbsp;</tspan></text></svg>'
))
monkeypatch.setenv("MERMAID_SVG_DIR", str(tmp_path))
svg = base64.b64encode(b'<svg id="mermaid-2"/>').decode()
assert mermaid_ssr.image(SOURCE, 2) == (
"\n\n![](data:image/svg+xml;base64,%s)\n\n" % svg)
markdown = mermaid_ssr.image(SOURCE, 2)
encoded = markdown.split("base64,", 1)[1].split(")", 1)[0]
svg = ET.fromstring(base64.b64decode(encoded))
assert svg.attrib["id"] == "mermaid-2"
assert svg.attrib["{http://www.w3.org/XML/1998/namespace}space"] == "preserve"
assert "".join(svg.itertext()) == "Bob's change 0.1\u00a0?\u00a0"
91 changes: 91 additions & 0 deletions contrib/test_pdf_layout.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
import copy
import json
from pathlib import Path
import subprocess


FILTER = Path(__file__).with_name('pdf-layout.lua')


def parse_markdown(markdown):
return json.loads(subprocess.run(
['pandoc', '-f', 'gfm', '-t', 'json'], input=markdown,
text=True, capture_output=True, check=True).stdout)


def apply_layout(document):
return json.loads(subprocess.run(
['pandoc', '-f', 'json', '-t', 'json', '--lua-filter', str(FILTER)],
input=json.dumps(document), text=True, capture_output=True,
check=True).stdout)


def only_table(document):
table, = [block['c'] for block in document['blocks'] if block['t'] == 'Table']
return table


def cell_text(cell):
return ''.join(inline['c'] if inline['t'] == 'Str' else ' '
for block in cell[4] for inline in block['c'])


def test_consecutive_notebook_images_each_get_their_own_block():
document = parse_markdown('![First plot](first.svg) ![Second plot](second.svg)')
block, = document['blocks']
block['t'] = 'Plain' # mdbook-pandoc emits notebook outputs as Plain.
original = [inline for inline in block['c'] if inline['t'] == 'Image']
result = apply_layout(document)
assert len(result['blocks']) == 2
assert all(block['t'] in ['Plain', 'Para'] for block in result['blocks'])
assert [block['c'] for block in result['blocks']] == [[image] for image in original]


def test_notebook_index_header_is_restored_without_shifting_column_names():
document = parse_markdown('''
| | set | value | binary (little endian) | popcount | weight (base 3) | weight (base 10) |
| --- | --- | --- | --- | --- | --- | --- |
| 0 | ternary | 0.00000486 | 011001111 | 6 | 1 | 3 |
| 1 | decimal | 0.00000500 | 001011111 | 6 | 4 | 1 |
''')
table = only_table(document)
del table[3][1][0][1][0] # mdbook-pandoc omits the empty index header.
result = only_table(apply_layout(document))
headers = result[3][1][0][1]
assert len(result[2]) == len(headers) == 7
assert [cell_text(cell) for cell in headers] == [
'', 'set', 'value', 'binary (little endian)', 'popcount',
'weight (base 3)', 'weight (base 10)',
]
rows = result[4][0][3]
assert [len(row[1]) for row in rows] == [7, 7]
assert [[cell_text(cell) for cell in row[1][:3]] for row in rows] == [
['0', 'ternary', '0.00000486'], ['1', 'decimal', '0.00000500'],
]


def test_bip_mixed_code_and_text_headers_keep_six_columns():
document = parse_markdown('''
| Name | `keytype` | `keydata` | `keydata` Description | `valuedata` | `valuedata` Description |
| --- | --- | --- | --- | --- | --- |
| Input Sort Key | `PSBT_IN_SORT_KEY = TBD` | None | No key data | `<bytes sort key>` | Orders inputs. |
''')
table = only_table(document)
headers = table[3][1][0][1]

def split_header(cell):
code, description = copy.deepcopy(cell), copy.deepcopy(cell)
code[4][0]['c'] = code[4][0]['c'][:1]
description[4][0]['c'] = description[4][0]['c'][1:]
return [code, description]

table[3][1][0][1] = (headers[:3] + split_header(headers[3])
+ [headers[4]] + split_header(headers[5]))
result = only_table(apply_layout(document))
assert len(result[2]) == len(result[3][1][0][1]) == 6
assert len(result[4][0][3][0][1]) == 6
restored = result[3][1][0][1]
for index in [3, 5]:
inlines = restored[index][4][0]['c']
assert len(inlines) > 1
assert inlines[-1] == {'t': 'Str', 'c': 'Description'}
69 changes: 69 additions & 0 deletions contrib/test_pdf_preprocess.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import json
from pathlib import Path
import subprocess
import sys

import pdf_preprocess


DATAFRAME = '''<div class="dataframe-wrapper">
<table border="1" class="dataframe">
<thead><tr><th></th><th>amount</th><th>label</th></tr></thead>
<tbody>
<tr><th>0</th><td>123</td><td>Alice &amp; Bob</td></tr>
<tr><th>1</th><td>456</td><td>change</td></tr>
</tbody>
</table>
</div>'''


def test_saved_dataframe_becomes_a_markdown_table_inside_its_wrapper():
content = 'Before output.\n\n' + DATAFRAME + '\n\nAfter output.'
converted = pdf_preprocess.convert_tables(content)
assert converted.startswith('Before output.\n\n<div class="dataframe-wrapper">')
assert converted.endswith('</div>\n\nAfter output.')
assert '<table' not in converted
parsed = json.loads(subprocess.run(
['pandoc', '-f', 'gfm', '-t', 'json'], input=converted,
text=True, capture_output=True, check=True).stdout)
table, = [block['c'] for block in parsed['blocks'] if block['t'] == 'Table']
assert len(table[2]) == 3
headers = table[3][1][0][1]
assert [cell[4] for cell in headers] == [
[],
[{'t': 'Plain', 'c': [{'t': 'Str', 'c': 'amount'}]}],
[{'t': 'Plain', 'c': [{'t': 'Str', 'c': 'label'}]}],
]
values = []
for row in table[4][0][3]:
values.append([
''.join(item.get('c', ' ') if item['t'] == 'Str' else ' '
for block in cell[4] for item in block['c'])
for cell in row[1]
])
assert values == [['0', '123', 'Alice & Bob'], ['1', '456', 'change']]


def test_walk_converts_only_notebook_chapters_including_children():
nested = {'Chapter': {'source_path': 'analysis.IPYNB', 'content': DATAFRAME,
'sub_items': []}}
markdown = {'Chapter': {'source_path': 'guide.md', 'content': DATAFRAME,
'sub_items': [nested]}}
pdf_preprocess.walk(['Separator', {'PartTitle': 'Part'}, markdown])
assert markdown['Chapter']['content'] == DATAFRAME
assert '<table' not in nested['Chapter']['content']


def test_json_protocol_and_renderer_support():
script = Path(pdf_preprocess.__file__)
for renderer, code in [('pandoc', 0), ('html', 1)]:
result = subprocess.run([sys.executable, script, 'supports', renderer])
assert result.returncode == code
book = {'sections': [{'Chapter': {
'source_path': 'analysis.ipynb', 'content': DATAFRAME, 'sub_items': []}}]}
for renderer in ['html', 'pandoc']:
result = subprocess.run([sys.executable, script],
input=json.dumps([{'renderer': renderer}, book]),
text=True, capture_output=True, check=True)
content = json.loads(result.stdout)['sections'][0]['Chapter']['content']
assert ('<table' not in content) == (renderer == 'pandoc')
Loading
Loading