diff --git a/.github/workflows/convert.yml b/.github/workflows/convert.yml index a400155..e013b65 100644 --- a/.github/workflows/convert.yml +++ b/.github/workflows/convert.yml @@ -38,9 +38,6 @@ jobs: - name: Install dependencies run: pip install -r requirements.txt - - name: Install pandoc - run: sudo apt-get update && sudo apt-get install -y pandoc - - name: Process content run: python scripts/process.py env: diff --git a/README.md b/README.md index b2ee590..ada0897 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ GitHub-hosted content shelf. Fork, upload, done. -> Fork this repo to get your own content platform on GitHub Pages. Upload PDFs or EPUBs (auto-converted to books), Markdown documents (rendered directly), or ZIP archives (deployed as static sites). Zero server cost. +> Fork this repo to get your own content platform on GitHub Pages. Upload PDFs or EPUBs (published as online books), Markdown documents (rendered directly), or ZIP archives (deployed as static sites). Zero server cost. ## Quick Start @@ -41,7 +41,7 @@ Your site is now live at `https://.github.io/gitshelf/` ([Create one here](https://github.com/settings/tokens/new?scopes=repo&description=GitShelf)) 3. Upload a file: - **`.pdf`** — Converted to a multi-chapter book via MinerU API - - **`.epub`** — Converted to a multi-chapter book via pandoc + - **`.epub`** — Stored as-is and read directly in the browser with TOC navigation - **`.md`** — Rendered directly as a document - **`.zip`** — Extracted as a static site (must contain `index.html`) 4. Wait for GitHub Actions to process (progress shown in Actions tab) @@ -68,7 +68,7 @@ Your site is now live at `https://.github.io/gitshelf/` Upload content (browser → GitHub API → input/) → GitHub Actions runs scripts/process.py → .pdf: MinerU API → Markdown → Split chapters → docs/books/{id}/ - → .epub: pandoc → Markdown + media → Split chapters → docs/books/{id}/ + → .epub: Store original EPUB + extract TOC → docs/books/{id}/ → .md: Copy to docs/articles/{id}/content.md → .zip: Extract to docs/sites/{id}/ → Build manifest → GitHub Pages deploys @@ -90,7 +90,7 @@ python -m unittest discover -s tests/scripts -v # Python pipeline tests **What if MinerU stops being free?** Swap it by modifying `scripts/mineru_client.py`. Works with any PDF-to-Markdown tool. -**Can I edit converted chapters?** Yes. Edit `.md` files in `docs/books//chapters/` and commit. +**Can I edit converted chapters?** Yes for PDF-converted books. Edit `.md` files in `docs/books//chapters/` and commit. EPUB books are read from the original `book.epub`. **Can I upload a static site?** Yes. Package it as a `.zip` with `index.html` at the root and upload through the admin panel. diff --git a/README.zh-CN.md b/README.zh-CN.md index 0280420..810a761 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -4,7 +4,7 @@ 基于 GitHub 的内容托管平台。Fork，上传，搞定。 -> Fork 本仓库即可拥有自己的内容平台。上传 PDF 或 EPUB（自动转换为书籍）、Markdown 文档（直接渲染）、ZIP 压缩包（部署为静态站点），全部托管在 GitHub Pages 上。零服务器成本。 +> Fork 本仓库即可拥有自己的内容平台。上传 PDF 或 EPUB（发布为在线书籍）、Markdown 文档（直接渲染）、ZIP 压缩包（部署为静态站点），全部托管在 GitHub Pages 上。零服务器成本。 ## 快速开始 @@ -41,7 +41,7 @@ （[点此创建](https://github.com/settings/tokens/new?scopes=repo&description=GitShelf)） 3. 上传文件： - **`.pdf`** — 通过 MinerU API 转换为多章节书籍 - - **`.epub`** — 通过 pandoc 转换为多章节书籍 + - **`.epub`** — 原样保存，并在浏览器中直接阅读，保留目录导航 - **`.md`** — 直接作为文档渲染展示 - **`.zip`** — 解压为静态站点（需包含 `index.html`） 4. 等待 GitHub Actions 处理完成 @@ -68,7 +68,7 @@ 上传内容（浏览器 → GitHub API → input/） → GitHub Actions 运行 scripts/process.py → .pdf: MinerU API → Markdown → 拆分章节 → docs/books/{id}/ - → .epub: pandoc → Markdown + 媒体资源 → 拆分章节 → docs/books/{id}/ + → .epub: 保存原始 EPUB + 提取目录 → docs/books/{id}/ → .md: 复制到 docs/articles/{id}/content.md → .zip: 解压到 docs/sites/{id}/ → 构建 manifest → GitHub Pages 部署 @@ -90,7 +90,7 @@ python -m unittest discover -s tests/scripts -v # Python 流水线测试 **MinerU 收费了怎么办？** 修改 `scripts/mineru_client.py` 即可替换，兼容任何 PDF 转 Markdown 工具。 -**可以手动编辑转换后的章节吗？** 可以。编辑 `docs/books//chapters/` 下的 `.md` 文件并提交即可。 +**可以手动编辑转换后的章节吗？** PDF 转换得到的书籍可以。编辑 `docs/books//chapters/` 下的 `.md` 文件并提交即可。EPUB 书籍则直接读取原始 `book.epub`。 **可以上传静态站点吗？** 可以。将站点打包为 `.zip`（根目录需包含 `index.html`），通过管理面板上传即可。 diff --git a/cli/github.js b/cli/github.js index 8f54b2e..fd8923c 100644 --- a/cli/github.js +++ b/cli/github.js @@ -105,6 +105,7 @@ function normalizeItem(raw) { visibility: normalizeVisibility(m.visibility), chapters_count: nullNum(m.chapters_count), word_count: nullNum(m.word_count), + source_format: String(m.source_format || '').trim() || null, created_at: String(m.created_at || '').trim() || null, updated_at: String(m.updated_at || m.created_at || '').trim() || null, source: String(m.source || '').trim() || null, @@ -241,7 +242,7 @@ function buildPublicManifest(items) { return { items: pub.map((b) => { const r = { id: b.id, type: b.type, title: getDisplayTitle(b), author: b.author || undefined, summary: b.summary || undefined, tags: b.tags, featured: b.featured, source: b.source, created_at: b.created_at, updated_at: b.updated_at }; - if (b.type === 'book') { r.chapters_count = b.chapters_count; r.word_count = b.word_count; } + if (b.type === 'book') { r.chapters_count = b.chapters_count; r.word_count = b.word_count; r.source_format = b.source_format || undefined; } else if (b.type === 'doc') { r.word_count = b.word_count; } else if (b.type === 'site') { r.entry = b.entry; } return r; @@ -258,7 +259,8 @@ function serializeCatalog(items) { author: b.author || '', summary: b.summary || '', tags: b.tags || [], featured: b.featured, manual_order: b.manual_order, visibility: normalizeVisibility(b.visibility), source: b.source || '', - entry: b.entry || undefined, chapters_count: b.chapters_count, word_count: b.word_count, + entry: b.entry || undefined, source_format: b.source_format || undefined, + chapters_count: b.chapters_count, word_count: b.word_count, created_at: b.created_at, updated_at: b.updated_at, })), }; diff --git a/package-lock.json b/package-lock.json index 07d2978..2ce46db 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,10 +10,12 @@ "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", + "epubjs": "^0.3.93", "zod": "^4.3.6" }, "bin": { - "gitshelf": "cli/index.js" + "gitshelf": "cli/index.js", + "gitshelf-mcp": "cli/mcp-server.mjs" }, "devDependencies": { "@preact/preset-vite": "^2.10.5", @@ -1277,6 +1279,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/localforage": { + "version": "0.0.34", + "resolved": "https://registry.npmjs.org/@types/localforage/-/localforage-0.0.34.tgz", + "integrity": "sha512-tJxahnjm9dEI1X+hQSC5f2BSd/coZaqbIl1m3TCl0q9SVuC52XcXfV0XmoCU1+PmjyucuVITwoTnN8OlTbEXXA==", + "deprecated": "This is a stub types definition for localforage (https://github.com/localForage/localForage). localforage provides its own type definitions, so you don't need @types/localforage installed!", + "license": "MIT", + "dependencies": { + "localforage": "*" + } + }, "node_modules/@vitest/expect": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.1.tgz", @@ -1400,6 +1412,16 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/@xmldom/xmldom": { + "version": "0.7.13", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.7.13.tgz", + "integrity": "sha512-lm2GW5PkosIzccsaZIz7tp8cPADSIlIHWDFTR1N0SzfinhhYgeIQjFMz4rYzanCScr3DqQLeomUDArp6MWKm+g==", + "deprecated": "this version has critical issues, please update to the latest version", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -1812,6 +1834,23 @@ "node": ">=6.6.0" } }, + "node_modules/core-js": { + "version": "3.49.0", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.49.0.tgz", + "integrity": "sha512-es1U2+YTtzpwkxVLwAFdSpaIMyQaq0PBgm3YD1W3Qpsn1NAmO3KSgZfu+oGSWVu6NvLHoHCV/aYcsE5wiB7ALg==", + "hasInstallScript": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, "node_modules/cors": { "version": "2.8.6", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", @@ -1894,6 +1933,19 @@ "dev": true, "license": "MIT" }, + "node_modules/d": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/d/-/d-1.0.2.tgz", + "integrity": "sha512-MOqHvMWF9/9MX6nza0KgvFH4HpMU0EF5uUDXqX/BtxtU8NfB0QzRtJ8Oe/6SuS4kbhyzVJwjd97EA4PKrzJ8bw==", + "license": "ISC", + "dependencies": { + "es5-ext": "^0.10.64", + "type": "^2.7.2" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/data-urls": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", @@ -2148,6 +2200,23 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/epubjs": { + "version": "0.3.93", + "resolved": "https://registry.npmjs.org/epubjs/-/epubjs-0.3.93.tgz", + "integrity": "sha512-c06pNSdBxcXv3dZSbXAVLE1/pmleRhOT6mXNZo6INKmvuKpYB65MwU/lO7830czCtjIiK9i+KR+3S+p0wtljrw==", + "license": "BSD-2-Clause", + "dependencies": { + "@types/localforage": "0.0.34", + "@xmldom/xmldom": "^0.7.5", + "core-js": "^3.18.3", + "event-emitter": "^0.3.5", + "jszip": "^3.7.1", + "localforage": "^1.10.0", + "lodash": "^4.17.21", + "marks-pane": "^1.0.9", + "path-webpack": "0.0.3" + } + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -2206,6 +2275,46 @@ "node": ">= 0.4" } }, + "node_modules/es5-ext": { + "version": "0.10.64", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.64.tgz", + "integrity": "sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==", + "hasInstallScript": true, + "license": "ISC", + "dependencies": { + "es6-iterator": "^2.0.3", + "es6-symbol": "^3.1.3", + "esniff": "^2.0.1", + "next-tick": "^1.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/es6-iterator": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", + "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==", + "license": "MIT", + "dependencies": { + "d": "1", + "es5-ext": "^0.10.35", + "es6-symbol": "^3.1.1" + } + }, + "node_modules/es6-symbol": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.4.tgz", + "integrity": "sha512-U9bFFjX8tFiATgtkJ1zg25+KviIXpgRvRHS8sau3GfhVzThRQrOeksPeT0BWW2MNZs1OEWJ1DPXOQMn0KKRkvg==", + "license": "ISC", + "dependencies": { + "d": "^1.0.2", + "ext": "^1.7.0" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -2222,6 +2331,21 @@ "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", "license": "MIT" }, + "node_modules/esniff": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/esniff/-/esniff-2.0.1.tgz", + "integrity": "sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==", + "license": "ISC", + "dependencies": { + "d": "^1.0.1", + "es5-ext": "^0.10.62", + "event-emitter": "^0.3.5", + "type": "^2.7.2" + }, + "engines": { + "node": ">=0.10" + } + }, "node_modules/estree-walker": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", @@ -2238,6 +2362,16 @@ "node": ">= 0.6" } }, + "node_modules/event-emitter": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", + "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==", + "license": "MIT", + "dependencies": { + "d": "1", + "es5-ext": "~0.10.14" + } + }, "node_modules/eventsource": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", @@ -2330,6 +2464,15 @@ "express": ">= 4.11" } }, + "node_modules/ext": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", + "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==", + "license": "ISC", + "dependencies": { + "type": "^2.7.2" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -2662,6 +2805,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, "node_modules/indent-string": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", @@ -3073,6 +3222,18 @@ "node": ">=6" } }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, "node_modules/katex": { "version": "0.16.42", "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.42.tgz", @@ -3097,6 +3258,15 @@ "dev": true, "license": "MIT" }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, "node_modules/lightningcss": { "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", @@ -3368,6 +3538,30 @@ "uc.micro": "^2.0.0" } }, + "node_modules/localforage": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/localforage/-/localforage-1.10.0.tgz", + "integrity": "sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==", + "license": "Apache-2.0", + "dependencies": { + "lie": "3.1.1" + } + }, + "node_modules/localforage/node_modules/lie": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz", + "integrity": "sha512-RiNhHysUjhrDQntfYSfY4MU24coXXdEOgw9WGcKHNeEwffDYbF//u87M1EWaMGzuFoSbqW0C9C6lEEhDOAswfw==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, "node_modules/lru-cache": { "version": "11.2.7", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.7.tgz", @@ -3436,6 +3630,12 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/marks-pane": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/marks-pane/-/marks-pane-1.0.9.tgz", + "integrity": "sha512-Ahs4oeG90tbdPWwAJkAAoHg2lRR8lAs9mZXETNPO9hYg3AkjUJBKi1NQ4aaIQZVGrig7c/3NUV1jANl8rFTeMg==", + "license": "MIT" + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -3549,6 +3749,12 @@ "node": ">= 0.6" } }, + "node_modules/next-tick": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", + "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==", + "license": "ISC" + }, "node_modules/node-html-parser": { "version": "6.1.13", "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.13.tgz", @@ -3681,6 +3887,12 @@ "wrappy": "1" } }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, "node_modules/parse5": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.0.tgz", @@ -3722,6 +3934,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/path-webpack": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/path-webpack/-/path-webpack-0.0.3.tgz", + "integrity": "sha512-AmeDxedoo5svf7aB3FYqSAKqMxys014lVKBzy1o/5vv9CtU7U4wgGWL1dA2o6MOzcD53ScN4Jmiq6VbtLz1vIQ==", + "license": "MIT" + }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -3836,6 +4054,12 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -3915,6 +4139,27 @@ "dev": true, "license": "MIT" }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/readable-stream/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, "node_modules/redent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", @@ -4009,6 +4254,12 @@ "node": ">= 18" } }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, "node_modules/safe-regex-test": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", @@ -4135,6 +4386,12 @@ "node": ">= 0.4" } }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -4318,6 +4575,15 @@ "node": ">= 0.4" } }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/strip-indent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz", @@ -4458,6 +4724,12 @@ "license": "0BSD", "optional": true }, + "node_modules/type": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/type/-/type-2.7.3.tgz", + "integrity": "sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==", + "license": "ISC" + }, "node_modules/type-is": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", @@ -4529,6 +4801,12 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", diff --git a/package.json b/package.json index 6f4744f..cc0c32a 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ }, "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", + "epubjs": "^0.3.93", "zod": "^4.3.6" } } diff --git a/scripts/build_manifest.py b/scripts/build_manifest.py index f2ebe71..4ad1d41 100644 --- a/scripts/build_manifest.py +++ b/scripts/build_manifest.py @@ -7,6 +7,7 @@ from pathlib import Path VALID_VISIBILITY = {"published", "hidden", "archived"} +EPUB_BOOK_FILENAME = "book.epub" def _utc_now_iso() -> str: @@ -152,6 +153,7 @@ def _to_public_item(entry: dict) -> dict: if entry["type"] == "book": result["chapters_count"] = entry.get("chapters_count") result["word_count"] = entry.get("word_count") + result["source_format"] = entry.get("source_format") elif entry["type"] == "doc": result["word_count"] = entry.get("word_count") elif entry["type"] == "site": @@ -171,16 +173,33 @@ def _build_book_entry(book_dir: Path) -> dict: chapters_dir = book_dir / "chapters" md_files = sorted(chapters_dir.glob("*.md")) if chapters_dir.is_dir() else [] + meta = _read_meta_json(book_dir / "meta.json") + source_format = str(meta.get("source_format", "")).strip() + if not source_format: + source_format = "epub" if (book_dir / EPUB_BOOK_FILENAME).exists() else "markdown-derived" + + def _count_toc_entries(items: list[dict]) -> int: + total = 0 + for item in items: + if item.get("slug") and not item.get("anchor"): + total += 1 + children = item.get("children") + if isinstance(children, list): + total += _count_toc_entries(children) + return total + chapters_count = len(md_files) word_count = sum(len(f.read_text(encoding="utf-8").split()) for f in md_files) + if chapters_count == 0: + chapters_count = _count_toc_entries(toc.get("children", [])) + word_count = None directory_modified_at = datetime.fromtimestamp( book_dir.stat().st_mtime, tz=timezone.utc ).strftime("%Y-%m-%dT%H:%M:%SZ") - meta = _read_meta_json(book_dir / "meta.json") - - source = str(meta.get("source", "")).strip() or f"{book_dir.name}.pdf" + default_ext = ".epub" if source_format == "epub" else ".pdf" + source = str(meta.get("source", "")).strip() or f"{book_dir.name}{default_ext}" updated_at = str(meta.get("updated_at", "")).strip() or directory_modified_at created_at = str(meta.get("created_at", "")).strip() or updated_at @@ -190,6 +209,7 @@ def _build_book_entry(book_dir: Path) -> dict: "generated_title": toc["title"], "chapters_count": chapters_count, "word_count": word_count, + "source_format": source_format, "created_at": created_at, "updated_at": updated_at, "source": source, @@ -288,6 +308,7 @@ def build_manifest( "metadata_updated_at": metadata.get("metadata_updated_at"), "chapters_count": generated["chapters_count"], "word_count": generated["word_count"], + "source_format": generated["source_format"], "created_at": generated["created_at"], "updated_at": generated["updated_at"], "source": metadata.get("source") or generated["source"], diff --git a/scripts/convert.py b/scripts/convert.py index 15832cf..15b8765 100644 --- a/scripts/convert.py +++ b/scripts/convert.py @@ -193,6 +193,7 @@ def _write_book_metadata( "id": book_id, "type": "book", "source": source_pdf, + "source_format": "markdown-derived", "pdf_md5": pdf_md5, "page_count": page_count, "created_at": created_at, diff --git a/scripts/process.py b/scripts/process.py index 0a102d3..00235ed 100644 --- a/scripts/process.py +++ b/scripts/process.py @@ -3,7 +3,7 @@ Handles four content types from input/: - .pdf → book (chapters via MinerU API) - - .epub → book (chapters via pandoc) + - .epub → book (raw EPUB + extracted TOC metadata) - .md → article (single markdown document) - .zip → site (static site extraction) @@ -14,27 +14,21 @@ import hashlib import json import os +import posixpath import re import shutil -import subprocess import sys -import tempfile import zipfile +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field from datetime import datetime, timezone -from pathlib import Path +from pathlib import Path, PurePosixPath try: from .build_manifest import build_manifest except ImportError: from build_manifest import build_manifest -try: - from .split_markdown import split_by_headings - from .generate_structure import generate_book_structure -except ImportError: - from split_markdown import split_by_headings - from generate_structure import generate_book_structure - # Reuse PDF pipeline from convert.py try: from .convert import ( @@ -43,7 +37,6 @@ ensure_unique_content_id, generate_book_id, reconvert_from_cache, - _rewrite_chapter_image_paths, _write_failures, _remove_failure, ) @@ -54,14 +47,24 @@ ensure_unique_content_id, generate_book_id, reconvert_from_cache, - _rewrite_chapter_image_paths, _write_failures, _remove_failure, ) FAILURES_FILENAME = "failures.json" BOOK_METADATA_FILENAME = "meta.json" -EPUB_CACHE_DIR = Path("cache/epub") +EPUB_BOOK_FILENAME = "book.epub" +EPUB_CONTAINER_PATH = "META-INF/container.xml" +EPUB_CONTAINER_NS = {"container": "urn:oasis:names:tc:opendocument:xmlns:container"} +EPUB_PACKAGE_NS = { + "opf": "http://www.idpf.org/2007/opf", + "dc": "http://purl.org/dc/elements/1.1/", +} +EPUB_NAV_NS = { + "xhtml": "http://www.w3.org/1999/xhtml", + "epub": "http://www.idpf.org/2007/ops", +} +EPUB_NCX_NS = {"ncx": "http://www.daisy.org/z3986/2005/ncx/"} def _utc_now_iso() -> str: @@ -100,6 +103,20 @@ def _read_existing_created_at(book_dir: Path, fallback: str) -> str: return str(existing.get("created_at", "")).strip() or fallback +def _read_existing_epub_checksum(book_dir: Path) -> str | None: + meta_path = book_dir / BOOK_METADATA_FILENAME + if not meta_path.exists(): + return None + + try: + existing = json.loads(meta_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return None + + checksum = str(existing.get("epub_md5", "")).strip() + return checksum or None + + def _write_epub_metadata( book_dir: Path, *, @@ -125,193 +142,292 @@ def _write_epub_metadata( ) -def _write_epub_cache(md5: str, epub_path: Path) -> None: - EPUB_CACHE_DIR.mkdir(parents=True, exist_ok=True) - shutil.copy2(epub_path, EPUB_CACHE_DIR / f"{md5}.epub") +def _slugify_epub(text: str) -> str: + normalized = text.strip().lower() + normalized = re.sub(r"[\s_]+", "-", normalized) + normalized = re.sub(r"[^\w-]+", "-", normalized, flags=re.UNICODE) + normalized = re.sub(r"-{2,}", "-", normalized).strip("-") + return normalized -def _find_cached_epub(output_dir: Path, source_epub: str) -> tuple[str, str] | None: - """Return ``(book_id, md5)`` for a cached EPUB source.""" - for meta_file in output_dir.glob(f"*/{BOOK_METADATA_FILENAME}"): - try: - meta = json.loads(meta_file.read_text(encoding="utf-8")) - except json.JSONDecodeError: - continue +@dataclass +class EpubTocEntry: + title: str + href: str + children: list["EpubTocEntry"] = field(default_factory=list) - if meta.get("source") == source_epub and meta.get("epub_md5"): - return meta_file.parent.name, str(meta["epub_md5"]) - return None +def _normalize_epub_href(base_path: str, href: str) -> str: + value = str(href or "").strip() + if not value: + return "" -def _sanitize_pandoc_markdown(markdown: str) -> str: - """Remove pandoc EPUB wrapper elements that add noise to rendered chapters.""" - sanitized = re.sub( - r"^\s*]*>\s*\s*$\n?", - "", - markdown, - flags=re.MULTILINE, - ) - sanitized = re.sub( - r"^\s*]*class=[\"'][^\"']*\bsection\b[^\"']*[\"'][^>]*>\s*$\n?", - "", - sanitized, - flags=re.MULTILINE, + path_part, frag = value.split("#", 1) if "#" in value else (value, "") + resolved = posixpath.normpath( + posixpath.join(posixpath.dirname(base_path), path_part) ) - sanitized = re.sub(r"^\s*\s*$\n?", "", sanitized, flags=re.MULTILINE) - sanitized = re.sub(r"\n{3,}", "\n\n", sanitized).strip() - return sanitized + "\n" if sanitized else "" + if frag: + return f"{resolved}#{frag}" + return resolved -def _rewrite_book_asset_paths( - markdown: str, - source_prefixes: tuple[str, ...], - dest_prefix: str, -) -> str: - """Rewrite relative markdown assets from one local prefix to another.""" +def _read_epub_package_path(zf: zipfile.ZipFile) -> str: + try: + container_xml = zf.read(EPUB_CONTAINER_PATH) + except KeyError as exc: + raise ValueError("EPUB is missing META-INF/container.xml") from exc - def _normalize_path(raw: str) -> str: - value = str(raw or "").strip() - if ( - not value - or value.startswith(("/", "#", "//", "data:")) - or re.match(r"^[a-z][a-z0-9+.-]*:", value, flags=re.IGNORECASE) - ): - return value + root = ET.fromstring(container_xml) + rootfile = root.find(".//container:rootfile", EPUB_CONTAINER_NS) + if rootfile is None: + raise ValueError("EPUB container.xml does not declare a package document") - for prefix in source_prefixes: - if value.startswith(prefix): - return f"{dest_prefix}{value[len(prefix):]}" + full_path = str(rootfile.attrib.get("full-path", "")).strip() + if not full_path: + raise ValueError("EPUB package document path is empty") + return full_path - return value - markdown = re.sub( - r"(!\[[^\]]*\]\()([^)]+)(\))", - lambda m: f"{m.group(1)}{_normalize_path(m.group(2))}{m.group(3)}", - markdown, - ) - markdown = re.sub( - r'(]*\bsrc=["\'])([^"\']+)(["\'][^>]*>)', - lambda m: f"{m.group(1)}{_normalize_path(m.group(2))}{m.group(3)}", - markdown, - flags=re.IGNORECASE, - ) - return markdown +def _parse_epub_nav_item( + li: ET.Element, + *, + nav_path: str, +) -> EpubTocEntry | None: + anchor = li.find("./xhtml:a", EPUB_NAV_NS) + if anchor is None: + anchor = li.find("./xhtml:span", EPUB_NAV_NS) + if anchor is None: + return None + title = " ".join("".join(anchor.itertext()).split()) + if not title: + return None -def _select_chapter_level(markdown: str) -> tuple[list, int]: - """Pick a usable heading level for book chapters, preferring multi-chapter splits.""" - fallback: tuple[list, int] | None = None - for level in (1, 2, 3): - try: - chapters = split_by_headings(markdown, level=level) - except ValueError: - continue + href = _normalize_epub_href(nav_path, anchor.attrib.get("href", "")) + children = [] + child_list = li.find("./xhtml:ol", EPUB_NAV_NS) + if child_list is not None: + for child in child_list.findall("./xhtml:li", EPUB_NAV_NS): + parsed = _parse_epub_nav_item(child, nav_path=nav_path) + if parsed is not None: + children.append(parsed) + + return EpubTocEntry(title=title, href=href, children=children) + + +def _extract_epub_nav_toc(zf: zipfile.ZipFile, nav_path: str) -> list[EpubTocEntry]: + root = ET.fromstring(zf.read(nav_path)) + toc_nav = None + for nav in root.findall(".//xhtml:nav", EPUB_NAV_NS): + nav_type = nav.attrib.get(f"{{{EPUB_NAV_NS['epub']}}}type", "") or nav.attrib.get("type", "") + if nav_type == "toc": + toc_nav = nav + break + + if toc_nav is None: + return [] + + toc_list = toc_nav.find("./xhtml:ol", EPUB_NAV_NS) + if toc_list is None: + return [] + + entries: list[EpubTocEntry] = [] + for item in toc_list.findall("./xhtml:li", EPUB_NAV_NS): + parsed = _parse_epub_nav_item(item, nav_path=nav_path) + if parsed is not None: + entries.append(parsed) + return entries + + +def _parse_epub_ncx_point(point: ET.Element, *, ncx_path: str) -> EpubTocEntry | None: + label = point.find("./ncx:navLabel/ncx:text", EPUB_NCX_NS) + content = point.find("./ncx:content", EPUB_NCX_NS) + if label is None or content is None: + return None + + title = " ".join("".join(label.itertext()).split()) + if not title: + return None - if fallback is None: - fallback = (chapters, level) + href = _normalize_epub_href(ncx_path, content.attrib.get("src", "")) + children = [] + for child in point.findall("./ncx:navPoint", EPUB_NCX_NS): + parsed = _parse_epub_ncx_point(child, ncx_path=ncx_path) + if parsed is not None: + children.append(parsed) - chapter_count = sum(1 for chapter in chapters if chapter.slug != "00-preface") - if chapter_count >= 2: - return chapters, level + return EpubTocEntry(title=title, href=href, children=children) - if fallback is None: - raise ValueError("No headings found in EPUB-derived markdown.") - return fallback +def _extract_epub_ncx_toc(zf: zipfile.ZipFile, ncx_path: str) -> list[EpubTocEntry]: + root = ET.fromstring(zf.read(ncx_path)) + nav_map = root.find("./ncx:navMap", EPUB_NCX_NS) + if nav_map is None: + return [] + entries: list[EpubTocEntry] = [] + for point in nav_map.findall("./ncx:navPoint", EPUB_NCX_NS): + parsed = _parse_epub_ncx_point(point, ncx_path=ncx_path) + if parsed is not None: + entries.append(parsed) + return entries -def _copy_epub_media(work_dir: Path, book_dir: Path) -> int: - """Copy pandoc-extracted media into the book's images directory.""" - media_dir = work_dir / "media" - if not media_dir.is_dir(): - return 0 - images_dir = book_dir / "images" - copied = 0 - for source in media_dir.rglob("*"): - if not source.is_file(): +def _guess_epub_fallback_toc( + spine: list[str], + manifest_by_id: dict[str, dict[str, str]], + *, + package_path: str, +) -> list[EpubTocEntry]: + entries: list[EpubTocEntry] = [] + for item_id in spine: + manifest_item = manifest_by_id.get(item_id) + if not manifest_item: continue - destination = images_dir / source.relative_to(media_dir) - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source, destination) - copied += 1 - return copied - - -def _run_pandoc_epub(epub_path: Path, work_dir: Path) -> str: - """Convert EPUB to Markdown using pandoc within a temporary work directory.""" - output_name = "book.md" - command = [ - "pandoc", - str(epub_path.resolve()), - "-t", - "gfm", - "--wrap=none", - "--extract-media=.", - "-o", - output_name, - ] + href = _normalize_epub_href(package_path, manifest_item.get("href", "")) + media_type = manifest_item.get("media_type", "") + if media_type not in {"application/xhtml+xml", "text/html"}: + continue + + title = PurePosixPath(href.split("#", 1)[0]).stem.replace("_", " ").replace("-", " ").strip() + entries.append(EpubTocEntry(title=title or item_id, href=href)) + return entries + + +def _build_epub_toc_data(epub_path: Path) -> tuple[str, list[EpubTocEntry]]: try: - subprocess.run( - command, - check=True, - cwd=work_dir, - capture_output=True, - text=True, - ) - except FileNotFoundError as exc: - raise RuntimeError( - "pandoc is required to process EPUB files. Install pandoc locally " - "and in GitHub Actions." - ) from exc - except subprocess.CalledProcessError as exc: - detail = (exc.stderr or exc.stdout or "").strip() - raise RuntimeError( - f"pandoc failed to convert {epub_path.name}: {detail[:200]}" - ) from exc - - output_path = work_dir / output_name - if not output_path.exists(): - raise RuntimeError(f"pandoc did not produce {output_name} for {epub_path.name}") - - return output_path.read_text(encoding="utf-8") - - -def _build_epub_book(epub_path: Path, output_dir: Path, book_id: str, title: str) -> int: - """Convert an EPUB source into the standard book directory structure.""" + with zipfile.ZipFile(epub_path, "r") as zf: + package_path = _read_epub_package_path(zf) + package_root = ET.fromstring(zf.read(package_path)) + except zipfile.BadZipFile as exc: + raise ValueError(f"Invalid EPUB file: {epub_path.name}: {exc}") from exc + + title = ( + package_root.findtext(".//dc:title", default="", namespaces=EPUB_PACKAGE_NS).strip() + or epub_path.stem + ) + + manifest_by_id: dict[str, dict[str, str]] = {} + nav_path = "" + ncx_path = "" + for item in package_root.findall(".//opf:manifest/opf:item", EPUB_PACKAGE_NS): + item_id = str(item.attrib.get("id", "")).strip() + href = str(item.attrib.get("href", "")).strip() + media_type = str(item.attrib.get("media-type", "")).strip() + properties = str(item.attrib.get("properties", "")).strip() + if not item_id or not href: + continue + + manifest_by_id[item_id] = { + "href": href, + "media_type": media_type, + "properties": properties, + } + if "nav" in properties.split(): + nav_path = _normalize_epub_href(package_path, href) + if media_type == "application/x-dtbncx+xml" and not ncx_path: + ncx_path = _normalize_epub_href(package_path, href) + + spine_element = package_root.find(".//opf:spine", EPUB_PACKAGE_NS) + spine_ids = [ + str(itemref.attrib.get("idref", "")).strip() + for itemref in package_root.findall(".//opf:spine/opf:itemref", EPUB_PACKAGE_NS) + if str(itemref.attrib.get("idref", "")).strip() + ] + + if spine_element is not None: + toc_id = str(spine_element.attrib.get("toc", "")).strip() + if toc_id and toc_id in manifest_by_id: + ncx_path = _normalize_epub_href(package_path, manifest_by_id[toc_id]["href"]) + + with zipfile.ZipFile(epub_path, "r") as zf: + toc_entries: list[EpubTocEntry] + if nav_path: + toc_entries = _extract_epub_nav_toc(zf, nav_path) + elif ncx_path: + toc_entries = _extract_epub_ncx_toc(zf, ncx_path) + else: + toc_entries = [] + + if not toc_entries: + toc_entries = _guess_epub_fallback_toc(spine_ids, manifest_by_id, package_path=package_path) + + if not toc_entries: + raise ValueError(f"Could not extract a table of contents from {epub_path.name}") + + return title, toc_entries + + +def _serialize_epub_toc_entries(entries: list[EpubTocEntry]) -> list[dict]: + used: dict[str, int] = {} + + def _convert(entry: EpubTocEntry) -> dict: + title_base = _slugify_epub(entry.title) + href_base = _slugify_epub(PurePosixPath(entry.href.split("#", 1)[0]).stem) + base = title_base or href_base or "section" + count = used.get(base, 0) + 1 + used[base] = count + slug = base if count == 1 else f"{base}-{count}" + + data = { + "title": entry.title, + "slug": slug, + "href": entry.href, + } + if entry.children: + data["children"] = [_convert(child) for child in entry.children] + return data + + return [_convert(entry) for entry in entries] + + +def _generate_epub_readme(title: str, toc_entries: list[dict]) -> str: + lines = [f"# {title}", "", "## Chapters", ""] + for entry in toc_entries: + lines.append(f"- {entry['title']}") + lines.extend( + [ + "", + "---", + "", + "> **Disclaimer:** This content is provided for personal study and research " + "purposes only. All rights belong to the original authors and copyright holders. " + "This is not an authorized distribution. If you are the rights holder and wish " + "to have this content removed, please contact the repository owner.", + "", + ] + ) + return "\n".join(lines) + + +def _build_epub_book(epub_path: Path, output_dir: Path, book_id: str) -> tuple[str, list[dict]]: + """Store the original EPUB and generate lightweight TOC metadata.""" + epub_bytes = epub_path.read_bytes() + title, raw_toc = _build_epub_toc_data(epub_path) + toc_entries = _serialize_epub_toc_entries(raw_toc) + book_dir = output_dir / book_id if book_dir.exists(): shutil.rmtree(book_dir) + book_dir.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix="gitshelf_epub_") as tmp_dir: - work_dir = Path(tmp_dir) - markdown = _run_pandoc_epub(epub_path, work_dir) - markdown = _sanitize_pandoc_markdown(markdown) - markdown = _rewrite_book_asset_paths( - markdown, - source_prefixes=("media/", "./media/"), - dest_prefix="images/", - ) - markdown = _rewrite_chapter_image_paths(markdown, book_id) - - chapters, chapter_level = _select_chapter_level(markdown) - generate_book_structure( - book_id, - title, - chapters, - chapter_level=chapter_level, - output_dir=output_dir, - ) - copied = _copy_epub_media(work_dir, book_dir) + (book_dir / EPUB_BOOK_FILENAME).write_bytes(epub_bytes) + (book_dir / "toc.json").write_text( + json.dumps({"title": title, "children": toc_entries}, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + (book_dir / "README.md").write_text( + _generate_epub_readme(title, toc_entries), + encoding="utf-8", + ) - return copied + return title, toc_entries def process_epub(epub_path: Path, output_dir: Path) -> None: - """Process a single .epub file into a multi-chapter book.""" + """Process a single .epub file into a raw EPUB-backed book.""" book_id = ensure_unique_content_id(_generate_id(epub_path), output_dir.parent, "book") - title = epub_path.stem md5 = _file_md5(epub_path) book_dir = output_dir / book_id timestamp = _utc_now_iso() @@ -319,11 +435,8 @@ def process_epub(epub_path: Path, output_dir: Path) -> None: print(f"Processing EPUB: {epub_path.name} -> {book_id}") - copied_assets = _build_epub_book(epub_path, output_dir, book_id, title) - if copied_assets: - print(f" Extracted {copied_assets} EPUB assets") - - _write_epub_cache(md5, epub_path) + title, toc_entries = _build_epub_book(epub_path, output_dir, book_id) + print(f" Extracted EPUB TOC with {len(toc_entries)} top-level entries") _write_epub_metadata( book_dir, @@ -339,28 +452,39 @@ def process_epub(epub_path: Path, output_dir: Path) -> None: def reconvert_epub_from_cache(source_epub: str, output_dir: Path) -> None: - """Rebuild an EPUB-backed book from cached source bytes.""" - cached = _find_cached_epub(output_dir, source_epub) - if not cached: + """Rebuild an EPUB-backed book from the stored raw EPUB source.""" + book_dir: Path | None = None + for meta_file in output_dir.glob(f"*/{BOOK_METADATA_FILENAME}"): + try: + meta = json.loads(meta_file.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + + if ( + meta.get("source") == source_epub + and meta.get("source_format") == "epub" + ): + book_dir = meta_file.parent + break + + if book_dir is None: raise FileNotFoundError( - f"No cached EPUB conversion found for {source_epub}. Re-upload the EPUB." + f"No stored EPUB source found for {source_epub}. Re-upload the EPUB." ) - book_id, md5 = cached - cached_epub = EPUB_CACHE_DIR / f"{md5}.epub" - if not cached_epub.exists(): + source_path = book_dir / EPUB_BOOK_FILENAME + if not source_path.exists(): raise FileNotFoundError( - f"Cached EPUB missing for MD5 {md5}. Re-upload the EPUB." + f"Stored EPUB missing at {source_path}. Re-upload the EPUB." ) - title = Path(source_epub).stem - book_dir = output_dir / book_id + book_id = book_dir.name + md5 = _file_md5(source_path) timestamp = _utc_now_iso() created_at = _read_existing_created_at(book_dir, timestamp) - print(f"Reconverting EPUB from cache: {source_epub} -> {book_id} (md5={md5})") - copied_assets = _build_epub_book(cached_epub, output_dir, book_id, title) - if copied_assets: - print(f" Extracted {copied_assets} EPUB assets") + print(f"Reconverting EPUB from stored source: {source_epub} -> {book_id} (md5={md5})") + title, toc_entries = _build_epub_book(source_path, output_dir, book_id) + print(f" Extracted EPUB TOC with {len(toc_entries)} top-level entries") _write_epub_metadata( book_dir, diff --git a/src/components/App.jsx b/src/components/App.jsx index c1fe207..0085a57 100644 --- a/src/components/App.jsx +++ b/src/components/App.jsx @@ -9,11 +9,12 @@ import { ScrollToTop } from './ScrollToTop'; import { ToastContainer } from './ToastContainer'; import { HomeView } from './HomeView'; import { ChapterReader } from './ChapterReader'; +import { EpubReader } from './EpubReader'; import { BookOverview } from './BookOverview'; import { ArticleReader } from './ArticleReader'; import { Footer } from './Footer'; import { SearchPanel } from './SearchPanel'; -import { fetchManifest } from '../lib/api'; +import { fetchManifest, getBookSourceFormat } from '../lib/api'; const AdminView = lazy(() => import('./AdminView')); @@ -26,10 +27,15 @@ export function App() { const [breadcrumb, setBreadcrumb] = useState(''); const [activeAnchor, setActiveAnchor] = useState(null); const [searchOpen, setSearchOpen] = useState(false); + const [activeBook, setActiveBook] = useState(undefined); + const [epubProgress, setEpubProgress] = useState(0); const isChapter = route.type === 'chapter'; const isBookRoute = route.type === 'book-overview' || route.type === 'chapter'; - const { progress, showScrollTop, scrollToTop } = useScrollProgress(isChapter); + const activeBookSourceFormat = activeBook === undefined ? null : getBookSourceFormat(activeBook); + const isEpubBook = activeBookSourceFormat === 'epub'; + const { progress: scrollProgress, showScrollTop, scrollToTop } = useScrollProgress(isChapter && !isEpubBook); + const progress = isEpubBook ? epubProgress : scrollProgress; const handleToggleSidebar = useCallback(() => { setSidebarOpen((prev) => !prev); @@ -48,6 +54,10 @@ export function App() { setActiveAnchor(anchor); }, []); + const handleEpubProgress = useCallback((nextProgress) => { + setEpubProgress(nextProgress); + }, []); + const handleOpenSearch = useCallback(() => setSearchOpen(true), []); const handleCloseSearch = useCallback(() => setSearchOpen(false), []); @@ -82,6 +92,32 @@ export function App() { } }, [route.type, route.siteId]); + useEffect(() => { + if (!isBookRoute) { + setActiveBook(undefined); + return undefined; + } + + let cancelled = false; + setActiveBook(undefined); + setEpubProgress(0); + fetchManifest() + .then((manifest) => { + if (cancelled) return; + const nextBook = manifest.items?.find((item) => item.id === route.bookId) || null; + setActiveBook(nextBook); + }) + .catch(() => { + if (!cancelled) { + setActiveBook(null); + } + }); + + return () => { + cancelled = true; + }; + }, [isBookRoute, route.bookId]); + // Reset sidebar state when leaving book routes useEffect(() => { if (!isBookRoute) { @@ -89,10 +125,13 @@ export function App() { setBreadcrumb(''); setSidebarOpen(false); setActiveAnchor(null); + setEpubProgress(0); } }, [isBookRoute]); const showSidebar = isBookRoute; + const showSearch = isBookRoute && activeBook !== undefined && activeBookSourceFormat !== 'epub'; + const waitingForBookFormat = route.type === 'chapter' && activeBook === undefined; return ( <> @@ -109,7 +148,7 @@ export function App() { showSidebarToggle={showSidebar} onToggleSidebar={handleToggleSidebar} sidebarExpanded={sidebarOpen} - showSearch={isBookRoute} + showSearch={showSearch} onSearch={handleOpenSearch} progress={progress} showProgress={isChapter} @@ -136,13 +175,26 @@ export function App() { /> )} {route.type === 'chapter' && ( - + waitingForBookFormat ? ( +
Loading book...
+ ) : isEpubBook ? ( + + ) : ( + + ) )} {route.type === 'article' && ( @@ -155,7 +207,7 @@ export function App() {