Repository navigation
Narrate the book with Kokoro #94
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
bc1cindy
wants to merge
3
commits into
main
Choose a base branch
from
narration
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+1,037
−6
Open
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| #!/usr/bin/env python3 | ||
| """An mdbook preprocessor that puts the narration's audio players in the book. | ||
|
|
||
| If TTS_AUDIO_DIR is set, it puts a block of audio elements at the top of | ||
| every chapter, one for each voice whose TTS_AUDIO_DIR/<voice>/<name>.mp3 | ||
| exists, as nix/tts.nix renders them, with the start of each section from | ||
| TTS_AUDIO_DIR/<voice>/<name>.json. A chapter whose text was not rendered, | ||
| like the generated contents page, gets none. Otherwise it changes nothing. | ||
| theme/narration.js turns the block into a floating player. | ||
|
|
||
| It lives apart from tts.py, which renders the audio, so that changing the | ||
| player does not re-render the book. | ||
| """ | ||
| import html | ||
| import json | ||
| import os | ||
| import sys | ||
| from pathlib import Path | ||
|
|
||
| import tts | ||
|
|
||
|
|
||
| def narrate(chapter, audio_dir, site_url): | ||
| """Put an audio player per rendered voice at the top of the chapter. | ||
|
|
||
| The player links its audio from SITE_URL, as the print page holds every | ||
| chapter at the root whatever the depth of its source. | ||
| """ | ||
| spoken = tts.text(chapter["content"]) | ||
| if not spoken: | ||
| return | ||
| mp3 = tts.name(spoken) + ".mp3" | ||
| players = [ | ||
| player(voice, mp3, site_url) | ||
| for voice in sorted(Path(audio_dir).iterdir()) | ||
| if (voice / mp3).exists() | ||
| ] | ||
| if players: | ||
| chapter["content"] = ( | ||
| '<div class="narration">\n' + "\n".join(players) + "\n</div>\n\n" + chapter["content"]) | ||
|
|
||
|
|
||
| def player(voice, mp3, site_url): | ||
| starts = (voice / mp3).with_suffix(".json") | ||
| sections = ( | ||
| f' data-sections="{html.escape(starts.read_text())}"' if starts.exists() else "") | ||
| return (f'<audio controls preload="none" title="{voice.name}"{sections} ' | ||
| f'src="{site_url}tts/{voice.name}/{mp3}"></audio>') | ||
|
|
||
|
|
||
| def walk_book(context, book): | ||
| audio_dir = os.environ.get("TTS_AUDIO_DIR") | ||
| if not audio_dir: | ||
| return | ||
| site_url = context["config"].get("output", {}).get("html", {}).get("site-url", "/") | ||
|
|
||
| def walk(items): | ||
| for item in items: | ||
| chapter = item.get("Chapter") if isinstance(item, dict) else None | ||
| if chapter: | ||
| narrate(chapter, audio_dir, site_url) | ||
| walk(chapter["sub_items"]) | ||
|
|
||
| # mdbook 0.5 renamed the top level key from `sections` to `items`. | ||
| walk(book.get("items", book.get("sections"))) | ||
|
|
||
|
|
||
| def main(): | ||
| if len(sys.argv) > 1 and sys.argv[1] == "supports": | ||
| sys.exit(0) | ||
| context, book = json.load(sys.stdin) | ||
| walk_book(context, book) | ||
| json.dump(book, sys.stdout) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| #!/usr/bin/env python3 | ||
| """Read a text aloud with Kokoro. | ||
|
|
||
| python3 speak.py TEXT VOICE OUT | ||
|
|
||
| reads the text file TEXT to the FLAC file OUT, taking the model from | ||
| KOKORO_CONFIG, KOKORO_MODEL and KOKORO_VOICES (a directory of <voice>.pt | ||
| files). It lives apart from tts.py so that changes there leave the rendered | ||
| sections cached. | ||
| """ | ||
| import os | ||
| import sys | ||
| from pathlib import Path | ||
|
|
||
| import numpy | ||
| import soundfile | ||
| import torch | ||
| from kokoro import KModel, KPipeline | ||
|
|
||
|
|
||
| def main(): | ||
| if len(sys.argv) != 4: | ||
| sys.exit(__doc__) | ||
| text, voice, out = Path(sys.argv[1]).read_text(), sys.argv[2], sys.argv[3] | ||
| model = KModel(repo_id="hexgrad/Kokoro-82M", | ||
| config=os.environ["KOKORO_CONFIG"], model=os.environ["KOKORO_MODEL"]).eval() | ||
| # The voice's first letter is its language: a for American, b for British | ||
| # English. | ||
| pipeline = KPipeline(lang_code=voice[0], repo_id="hexgrad/Kokoro-82M", model=model) | ||
| pack = torch.load(Path(os.environ["KOKORO_VOICES"]) / (voice + ".pt"), weights_only=True) | ||
| audio = [a for _, _, a in pipeline(text, voice=pack, speed=1) if a is not None] | ||
| soundfile.write(out, numpy.concatenate(audio), 24000, format="FLAC") | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| import narration | ||
| import tts | ||
|
|
||
|
|
||
| CHAPTER = "## Trust scope\n\nEvery PSBT role inside it may observe the full PSBT.\n" | ||
|
|
||
|
|
||
| def render(audio_dir, voice, content): | ||
| mp3 = tts.name(tts.text(content)) + ".mp3" | ||
| (audio_dir / voice).mkdir(parents=True, exist_ok=True) | ||
| (audio_dir / voice / mp3).write_bytes(b"mp3") | ||
| return mp3 | ||
|
|
||
|
|
||
| def chapter(content): | ||
| return {"name": "Scrubber", "content": content, "sub_items": []} | ||
|
|
||
|
|
||
| CONTEXT = {"config": {"output": {"html": {"site-url": "/docs/"}}}} | ||
|
|
||
|
|
||
| def test_one_player_per_rendered_voice(tmp_path): | ||
| mp3 = render(tmp_path, "af_heart", CHAPTER) | ||
| render(tmp_path, "bf_emma", CHAPTER) | ||
| item = chapter(CHAPTER) | ||
| narration.narrate(item, tmp_path, "/docs/") | ||
| assert item["content"] == ( | ||
| '<div class="narration">\n' | ||
| f'<audio controls preload="none" title="af_heart" src="/docs/tts/af_heart/{mp3}"></audio>\n' | ||
| f'<audio controls preload="none" title="bf_emma" src="/docs/tts/bf_emma/{mp3}"></audio>\n' | ||
| '</div>\n\n' | ||
| + CHAPTER) | ||
|
|
||
|
|
||
| def test_player_carries_the_section_starts(tmp_path): | ||
| mp3 = render(tmp_path, "af_heart", CHAPTER) | ||
| (tmp_path / "af_heart" / mp3).with_suffix(".json").write_text( | ||
| '[{"title": "Trust scope", "start": 0.0}]') | ||
| item = chapter(CHAPTER) | ||
| narration.narrate(item, tmp_path, "/docs/") | ||
| assert ('<audio controls preload="none" title="af_heart" data-sections=' | ||
| '"[{"title": "Trust scope", "start": 0.0}]" ' | ||
| f'src="/docs/tts/af_heart/{mp3}"></audio>') in item["content"] | ||
|
|
||
|
|
||
| def test_text_not_rendered_gets_no_player(tmp_path): | ||
| render(tmp_path, "af_heart", "Other text.") | ||
| item = chapter(CHAPTER) | ||
| narration.narrate(item, tmp_path, "/docs/") | ||
| assert item["content"] == CHAPTER | ||
|
|
||
|
|
||
| def test_no_player_for_a_chapter_without_text(tmp_path): | ||
| (tmp_path / "af_heart").mkdir() | ||
| item = chapter("```mermaid\nA\n```\n") | ||
| narration.narrate(item, tmp_path, "/docs/") | ||
| assert item["content"] == "```mermaid\nA\n```\n" | ||
|
|
||
|
|
||
| def test_walk_defaults_to_the_root_without_site_url(tmp_path, monkeypatch): | ||
| mp3 = render(tmp_path, "af_heart", CHAPTER) | ||
| monkeypatch.setenv("TTS_AUDIO_DIR", str(tmp_path)) | ||
| book = {"items": [{"Chapter": chapter(CHAPTER)}]} | ||
| narration.walk_book({"config": {}}, book) | ||
| assert f'src="/tts/af_heart/{mp3}"' in book["items"][0]["Chapter"]["content"] | ||
|
|
||
|
|
||
| def test_walk_without_audio_dir_changes_nothing(monkeypatch): | ||
| monkeypatch.setenv("TTS_AUDIO_DIR", "") | ||
| book = {"items": [{"Chapter": chapter(CHAPTER)}, "Separator"]} | ||
| narration.walk_book(CONTEXT, book) | ||
| assert book["items"][0]["Chapter"]["content"] == CHAPTER | ||
|
|
||
|
|
||
| def test_walk_reaches_sub_chapters(tmp_path, monkeypatch): | ||
| render(tmp_path, "af_heart", CHAPTER) | ||
| monkeypatch.setenv("TTS_AUDIO_DIR", str(tmp_path)) | ||
| child = chapter(CHAPTER) | ||
| book = {"items": [{"Chapter": dict(chapter("Short."), sub_items=[{"Chapter": child}])}]} | ||
| narration.walk_book(CONTEXT, book) | ||
| assert child["content"].startswith('<div class="narration">') |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| import tts | ||
|
|
||
|
|
||
| def test_drops_frontmatter(): | ||
| assert tts.text("---\ntitle: Scrubber\nstatus: draft\n---\n\nBody text.\n") == "Body text." | ||
|
|
||
|
|
||
| def test_drops_fenced_blocks_of_any_kind(): | ||
| markdown = ( | ||
| "Before.\n\n" | ||
| "```mermaid\nsequenceDiagram\n A->>B: hi\n```\n\n" | ||
| "````rust\nfn main() {}\n````\n\n" | ||
| "~~~\nplain\n~~~\n\n" | ||
| "After.\n") | ||
| assert tts.text(markdown) == "Before.\nAfter." | ||
|
|
||
|
|
||
| def test_drops_html_and_math(): | ||
| markdown = ( | ||
| "<div class=\"note\">\n<p>Hidden</p>\n</div>\n\n" | ||
| "A fee of $f$ per input.\n\n" | ||
| "$$\nx = y\n$$\n\n" | ||
| "Text with <br> a tag.\n") | ||
| assert tts.text(markdown) == "A fee of per input.\nText with a tag." | ||
|
|
||
|
|
||
| def test_keeps_link_and_inline_code_text(): | ||
| markdown = "See [BIP 370](https://example.com/370) and the `PSBT_IN_SIGHASH` field.\n" | ||
| assert tts.text(markdown) == "See BIP 370 and the PSBT_IN_SIGHASH field." | ||
|
|
||
|
|
||
| def test_drops_images_tables_and_footnote_definitions(): | ||
| markdown = ( | ||
| "Intro.[^1]\n\n" | ||
| "\n\n" | ||
| "| Field | Kept |\n|---|---|\n| a | b |\n\n" | ||
| "[^1]: A note.\n") | ||
| assert tts.text(markdown) == "Intro." | ||
|
|
||
|
|
||
| def test_drops_tables_without_outer_pipes(): | ||
| markdown = ( | ||
| "Fields:\n\n" | ||
| "`<keytype>` | PSBT version | BIP |\n" | ||
| "--- | --- | --- |\n" | ||
| "`PSBT_GLOBAL_XPUB = 0x01` | 0, 2 | [174](https://example.com) |\n\n" | ||
| "After.\n") | ||
| assert tts.text(markdown) == "Fields:\nAfter." | ||
|
|
||
|
|
||
| def test_headings_and_list_items_are_units(): | ||
| markdown = "## Trust scope\n\n- Creator\n- Signer\n\n1. First\n2. Second\n" | ||
| assert tts.text(markdown) == "Trust scope.\nCreator.\nSigner.\nFirst.\nSecond." | ||
|
|
||
|
|
||
| def test_wrapped_paragraph_lines_join(): | ||
| assert tts.text("One sentence\nwrapped over\nthree lines.\n") == ( | ||
| "One sentence wrapped over three lines.") | ||
|
|
||
|
|
||
| def test_no_text(): | ||
| assert tts.text("---\ntitle: Stub\n---\n\n```mermaid\nA\n```\n") == "" | ||
|
|
||
|
|
||
| def test_name_depends_only_on_text(): | ||
| assert tts.name("Hello.") == tts.name("Hello.") | ||
| assert tts.name("Hello.") != tts.name("Hello!") | ||
| assert len(tts.name("Hello.")) == 16 | ||
|
|
||
|
|
||
| def test_sections_cut_before_headings(): | ||
| markdown = "Lead.\n\n## Trust scope\n\nFirst.\n\n- Item\n\n### Roles\n\nSecond.\n" | ||
| assert tts.sections(markdown) == [ | ||
| {"title": "", "level": 0, "text": "Lead."}, | ||
| {"title": "Trust scope", "level": 2, "text": "Trust scope.\nFirst.\nItem."}, | ||
| {"title": "Roles", "level": 3, "text": "Roles.\nSecond."}, | ||
| ] | ||
|
|
||
|
|
||
| def test_sections_join_to_the_text(): | ||
| markdown = "# Title\n\nOne.\n\n```\n# not a heading\n```\n\n## Next\n\nTwo.\n" | ||
| assert "\n".join(s["text"] for s in tts.sections(markdown)) == tts.text(markdown) | ||
| assert [s["title"] for s in tts.sections(markdown)] == ["Title", "Next"] | ||
|
|
||
|
|
||
| def test_no_sections_without_text(): | ||
| assert tts.sections("```mermaid\nA\n```\n") == [] | ||
|
|
||
|
|
||
| def test_starts_add_up_durations(): | ||
| cut = [{"title": t, "level": 2, "text": ""} for t in "ABC"] | ||
| assert tts.starts(cut, [1.5, 2.25, 3]) == [ | ||
| {"title": "A", "level": 2, "start": 0.0}, | ||
| {"title": "B", "level": 2, "start": 1.5}, | ||
| {"title": "C", "level": 2, "start": 3.75}, | ||
| ] |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.