diff --git a/astro.config.mjs b/astro.config.mjs index 17e8364..f773844 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url'; import { defineConfig } from 'astro/config'; import mdx from '@astrojs/mdx'; import { rehypeCjkSegmentBreaks } from './scripts/lib/cjk-segment-break.mjs'; +import { remarkCjkQuotes } from './scripts/lib/cjk-quotes.mjs'; import sitemap from '@astrojs/sitemap'; import { readPostLastmods } from './scripts/lib/post-dates.mjs'; @@ -99,7 +100,11 @@ export default defineConfig({ // Chromium and Safari; this removes it from the rendered prose (code // excluded). MDX inherits `markdown` config. See scripts/lib/cjk-segment-break.mjs, // and `pnpm content:lint --dist`, which fails the build on any that remain. - markdown: { shikiConfig: { theme: 'github-dark-default' }, rehypePlugins: [rehypeCjkSegmentBreaks] }, + // Smartypants curls quotes by their neighbours, which points the opening mark + // of a zh or ja pair the wrong way; `remarkCjkQuotes` runs after it and sets + // each mark by its position in its pair, in zh-Hans, zh-Hant, ja and ko files + // only. See scripts/lib/cjk-quotes.mjs; `content:lint --dist` guards it too. + markdown: { shikiConfig: { theme: 'github-dark-default' }, remarkPlugins: [remarkCjkQuotes], rehypePlugins: [rehypeCjkSegmentBreaks] }, integrations: [ mdx(), sitemap({ diff --git a/scripts/content-lint.mjs b/scripts/content-lint.mjs index a1c150b..e70519a 100644 --- a/scripts/content-lint.mjs +++ b/scripts/content-lint.mjs @@ -20,6 +20,7 @@ import { isCjkWide, lastChar, } from './lib/cjk-segment-break.mjs'; +import { OPAQUE_INLINE, isApostrophe } from './lib/cjk-quotes.mjs'; const ROOT = cwd(); const BLOG = path.join(ROOT, 'content', 'blog'); @@ -571,11 +572,201 @@ async function checkBuiltCjkBreaks() { return 1; } +// ─── Built HTML: quotation marks in zh, ja and ko prose point the right way ─ +// +// Smartypants curls an ASCII quote by its neighbours, so in Chinese, which puts +// no space before a quotation, the opening mark of a pair after a character +// comes out closing: 「大多数”给你的业务加上 AI”的说辞」. It also reads a whole +// document as one string, so a heading or table cell that starts with a quote +// takes its direction from the block before it (the Korean headings). The +// remark step in astro.config.mjs sets each mark by its position in its pair; +// this reads what the build actually emitted, on every zh, ja and ko page, so +// it also fails when that step is not wired, or when CJK prose reaches a page +// by a road the step never sees (an .astro template, a data file). +// +// The rule is the step's own, defined once in scripts/lib/cjk-quotes.mjs: in +// one block of prose, curly double marks alternate “ ” and end closed, and so +// do single marks, except an apostrophe (`isApostrophe`), which is always ’. +// The HTML scanning below is this gate's own. Straight ASCII marks have no +// direction and are not read; marks inside code, pre, kbd and samp are not +// prose and are not read either. + +/** Elements that sit inside a sentence without ending its block of prose. */ +const QUOTE_INLINE_ELEMENTS = new Set([...TRANSPARENT_ELEMENTS, 'br', 'wbr', 'img']); + +/** + * Blocks of prose: text joined across inline elements and cut at every other + * tag. An opaque element (code, kbd, svg, …) is one `OPAQUE_INLINE` in the + * block, as it is to the remark step: a quotation may wrap inline code. + */ +function quoteBlocks(html) { + const blocks = []; + let block = ''; + let opaqueDepth = 0; + const cut = () => { + if (block) blocks.push(block); + block = ''; + }; + let i = 0; + while (i < html.length) { + const lt = html.indexOf('<', i); + const stop = lt === -1 ? html.length : lt; + if (stop > i && opaqueDepth === 0) block += decodeEntities(html.slice(i, stop)); + if (lt === -1) break; + if (html.startsWith('', lt + 4); + i = end === -1 ? html.length : end + 3; + continue; + } + if (html.startsWith(' 0) continue; + if (name === 'br') block += '\n'; + else if (name === 'img') block += OPAQUE_INLINE; + else if (!QUOTE_INLINE_ELEMENTS.has(name)) cut(); + } + cut(); + return blocks; +} + +/** + * Every curly mark in one block whose direction disagrees with its position + * in its pair, plus a pair left open at the end of the block. Returns the + * number of marks read and the hits, each with its sentence for the report. + */ +function misdirectedQuotes(block) { + const hits = []; + let marks = 0; + let doubleOpen = false; + let singleOpen = false; + // Where the open pair started, or -1 when its opening mark was already + // reported as misdirected (a lone `”` is one problem, not two). + let doubleAt = -1; + let singleAt = -1; + const context = (at, problem) => + `${problem}: …${block.slice(Math.max(0, at - 14), at)}⟦${block[at] ?? ''}⟧${block.slice(at + 1, at + 15)}…` + .replace(/\s+/g, ' ') + .replaceAll(OPAQUE_INLINE, '▯'); + for (let i = 0; i < block.length; i++) { + const ch = block[i]; + if (ch === '“' || ch === '”') { + marks++; + const expected = doubleOpen ? '”' : '“'; + if (ch !== expected) hits.push(context(i, doubleOpen ? 'a closing mark printed as “' : 'an opening mark printed as ”')); + doubleOpen = !doubleOpen; + if (doubleOpen) doubleAt = ch === expected ? i : -1; + } else if (ch === '‘' || ch === '’') { + marks++; + if (isApostrophe(lastChar(block.slice(0, i)), block.slice(i + 1, i + 6), singleOpen)) { + if (ch === '‘') hits.push(context(i, 'an apostrophe printed as ‘')); + continue; + } + const expected = singleOpen ? '’' : '‘'; + if (ch !== expected) hits.push(context(i, singleOpen ? 'a closing mark printed as ‘' : 'an opening mark printed as ’')); + singleOpen = !singleOpen; + if (singleOpen) singleAt = ch === expected ? i : -1; + } + } + if (doubleOpen && doubleAt !== -1) hits.push(context(doubleAt, 'a “ never closed in its block')); + if (singleOpen && singleAt !== -1) hits.push(context(singleAt, 'a ‘ never closed in its block')); + return { marks, hits }; +} + +/** A page whose document language is Chinese, Japanese or Korean. */ +const CJK_QUOTE_LANG = /]*\blang\s*=\s*["']?(?:zh|ja|ko)(?:-|["'\s>])/i; + +async function checkBuiltCjkQuotes() { + const pages = (await distFiles(DIST)).filter((file) => file.endsWith('.html')).sort(); + const offenders = []; + let cjkPages = 0; + let marks = 0; + let total = 0; + for (const page of pages) { + const html = await readFile(path.join(DIST, page), 'utf8'); + if (!CJK_QUOTE_LANG.test(html)) continue; + cjkPages++; + const hits = []; + for (const block of quoteBlocks(html)) { + const result = misdirectedQuotes(block); + marks += result.marks; + hits.push(...result.hits); + } + total += hits.length; + if (hits.length > 0) offenders.push({ page, hits }); + } + if (cjkPages === 0 || marks === 0) { + console.error( + `✗ content lint --dist found ${cjkPages === 0 ? 'no zh, ja or ko page (by ``)' : 'no curly quotation mark on any zh, ja or ko page'} ` + + `among ${pages.length} built page${pages.length === 1 ? '' : 's'} under ${path.relative(ROOT, DIST)}/.\n` + + ` Those are the pages where a quotation mark can point the wrong way, so this checked nothing, ` + + `which is not the same as nothing being wrong.` + ); + return 1; + } + if (offenders.length === 0) { + console.log( + `✓ content lint --dist CJK quotation marks passed (${cjkPages} zh/ja/ko pages, ${marks} curly marks, ` + + `0 pointing the wrong way or left open)` + ); + return 0; + } + for (const { page, hits } of offenders) { + console.log(`✗ ${page}: ${hits.length} quotation mark${hits.length === 1 ? '' : 's'} pointing the wrong way or left open`); + for (const hit of hits.slice(0, 5)) console.log(` ${hit}`); + if (hits.length > 5) console.log(` … and ${hits.length - 5} more`); + } + console.error( + `\n✗ content lint --dist failed (${total} quotation mark${total === 1 ? '' : 's'} pointing the wrong way or left open on ` + + `${offenders.length} zh/ja/ko page${offenders.length === 1 ? '' : 's'}; ⟦ ⟧ marks each)\n` + + ` In one paragraph, heading or table cell, the first mark of a pair opens and the next closes.\n` + + ` Markdown and MDX in zh-Hans, zh-Hant, ja and ko are set by \`remarkCjkQuotes\` (scripts/lib/cjk-quotes.mjs),\n` + + ` wired in astro.config.mjs \`markdown.remarkPlugins\`: if these pages are posts, that step did not run, or a\n` + + ` paragraph holds an odd number of marks (a stray \`"\`, or a quotation that runs over two paragraphs; close it\n` + + ` in each). If the text came from somewhere else (an .astro template, a data file), fix the marks in that source.` + ); + return 1; +} + if (args.has('--dist')) { const prose = await checkBuiltProse(); const cjkBreaks = await checkBuiltCjkBreaks(); + const cjkQuotes = await checkBuiltCjkQuotes(); const links = await checkBuiltLinks(); - exit(prose || cjkBreaks || links); + exit(prose || cjkBreaks || cjkQuotes || links); } // The frontmatter taxonomy, derived from the module that declares it instead of diff --git a/scripts/lib/cjk-quotes.mjs b/scripts/lib/cjk-quotes.mjs new file mode 100644 index 0000000..375145c --- /dev/null +++ b/scripts/lib/cjk-quotes.mjs @@ -0,0 +1,197 @@ +// Quotation marks in CJK prose must point the right way. +// +// Astro's default smartypants (remark-smartypants over retext-smartypants) +// curls an ASCII quote by looking at the characters around it: after a letter +// it is a closing mark, after a space or punctuation an opening one. Chinese +// and Japanese put no space before a quotation, so the opening mark of a pair +// that follows a character comes out closing: 「大多数”给你的业务加上 AI”的说辞」. +// It also reads the whole document as one string, so a heading or table cell +// that starts with a quote takes its direction from the last letter of the +// block before it; that is the Korean case, where words are spaced +// («”다른 AI에게 검토를 맡기면 되지”가 …» as an h2). An audit of the built site +// found 299 such pairs in 16 zh-Hans posts, the same in their zh-Hant copies, +// and 13 headings and table cells in six ko posts. +// +// The rule here does not guess from neighbours. Within one block of prose (a +// paragraph, a heading, a table cell) double quotation marks come in pairs, so +// a mark's direction is its position in the pair: the first opens, the next +// closes. A mark the author already typed curly counts the same way, so a +// source that mixes `"` and `“ ”` still pairs. Single marks pair the same way, +// except an apostrophe, which is not part of a pair (see `isApostrophe`). +// +// Two consumers share these definitions, so they cannot disagree: +// +// * `remarkCjkQuotes`, the build-time fix, wired into +// `markdown.remarkPlugins` in astro.config.mjs (MDX inherits it). It runs +// after smartypants and only on zh-Hans, zh-Hant, ja and ko files, chosen +// by file name, so an English, German, Spanish or French page is not +// touched at all. Every future post in those locales is covered, however +// its quotes were typed. +// * `pnpm content:lint --dist`, the guard, which reads the BUILT HTML of +// every zh, ja and ko page with its own scanner and fails on a mark whose +// direction disagrees with its position, or a pair left open. It catches +// the step not running, and CJK prose reaching a page by another road. +// +// The WeChat export (scripts/lib/wechat-html.mjs) renders with marked, which +// does not curl quotes at all, and uses `directQuotes` on its own tokens. +// +// Code is never touched: inline code, images and other non-prose inlines are +// opaque. They sit in the sentence (a quote can wrap inline code), but their +// text is neither edited nor read. +// +// The marks stay “ ” and ‘ ’. AGENTS.md records no zh quotation style, and the +// sources already type “ ” (354 pairs in zh-Hans) far more than 「」 (23), so +// the author's choice of mark is kept and only the direction is corrected; +// a source that types 「」 is left exactly as written. + +import { OPAQUE_ELEMENTS } from './cjk-segment-break.mjs'; + +/** Locales whose prose puts no space before a quotation, or whose files this step owns. */ +export const CJK_QUOTE_LOCALES = new Set(['zh-Hans', 'zh-Hant', 'ja', 'ko']); + +/** + * The CJK locale of a content file from its name, or null. A translation is + * `index..mdx` (or flat `..mdx`); the English original + * carries no locale and is never matched. + */ +export function cjkLocaleOfPath(file) { + const m = /\.([A-Za-z]{2,3}(?:-[A-Za-z]{2,4})?)\.mdx?$/.exec(String(file ?? '')); + return m && CJK_QUOTE_LOCALES.has(m[1]) ? m[1] : null; +} + +export const DOUBLE_QUOTES = new Set(['"', '“', '”']); +export const SINGLE_QUOTES = new Set(["'", '‘', '’']); + +/** + * Stands for an opaque inline (inline code, an image) in a block's text. It + * counts as a word character, so `` `x`'s `` keeps its apostrophe. + */ +export const OPAQUE_INLINE = '\uFFFC'; + +const WORD_BEFORE = /^[\p{Script=Latin}\p{Nd}\uFFFC]$/u; +const LETTER_AFTER = /^\p{Script=Latin}/u; +// `'26`, `'90s`: the century is elided, so the mark is an apostrophe. Only +// when the year ends there (a space, the end, or sentence punctuation): a +// quotation that opens on a number, `'40~60% 절약'`, is still a quotation. +const DECADE_AFTER = /^\d\ds?(?=$|\s|[,，、;；:：)）!！?？]|[.。](?!\d))/u; + +/** + * True when a single mark is an apostrophe rather than a quotation mark: it + * follows a Latin letter or digit and either a letter follows (`don't`) or no + * single quotation is open (`users'`), or it elides a century (`Next '26,`). An + * apostrophe is always ’ and never opens or closes a pair. The test needs only + * the character before and the text after, because ’ is also what a closing + * mark looks like: getting a possessive wrong for a closing mark is invisible. + */ +export function isApostrophe(before, after, singleOpen) { + if (WORD_BEFORE.test(before) && (LETTER_AFTER.test(after) || !singleOpen)) return true; + return DECADE_AFTER.test(after); +} + +function lastCharOf(text) { + const chars = Array.from(text.slice(-2)); + return chars[chars.length - 1] ?? ''; +} + +/** + * Directs every quotation mark in one block of prose. `segments` is the + * block's text in document order: a string is prose, `null` an opaque inline. + * Returns the segments with each double and single mark set by its position + * in its pair; nothing else changes, and `null` stays `null`. + */ +export function directQuotes(segments) { + const full = segments.map((s) => (s === null ? OPAQUE_INLINE : s)).join(''); + let doubleOpen = false; + let singleOpen = false; + let at = 0; + return segments.map((segment) => { + if (segment === null) { + at += OPAQUE_INLINE.length; + return null; + } + let out = ''; + for (const ch of segment) { + if (DOUBLE_QUOTES.has(ch)) { + out += doubleOpen ? '”' : '“'; + doubleOpen = !doubleOpen; + } else if (SINGLE_QUOTES.has(ch)) { + const before = lastCharOf(full.slice(Math.max(0, at - 2), at)); + if (isApostrophe(before, full.slice(at + 1, at + 6), singleOpen)) { + out += '’'; + } else { + out += singleOpen ? '’' : '‘'; + singleOpen = !singleOpen; + } + } else { + out += ch; + } + at += ch.length; + } + return out; + }); +} + +/** True when the text holds a CJK ideograph, kana or Hangul syllable. */ +export function hasCjkLetter(text) { + return /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u.test(text); +} + +// ─── remark ────────────────────────────────────────────────────────────── + +/** mdast nodes that hold one block of prose: quotation pairs never span two. */ +const BLOCKS = new Set(['paragraph', 'heading', 'tableCell']); + +/** Inline mdast nodes whose children are part of the same sentence. */ +const CONTAINERS = new Set(['emphasis', 'strong', 'delete', 'link', 'linkReference']); + +/** + * The block's inline content in order: text nodes, `null` for an opaque + * inline (inline code, an image, raw HTML, an MDX expression), and a line + * break as the string it stands for. + */ +function collect(node, items) { + for (const child of node.children ?? []) { + if (child.type === 'text') { + items.push(child); + } else if (child.type === 'break') { + items.push('\n'); + } else if (CONTAINERS.has(child.type)) { + collect(child, items); + } else if ( + child.type === 'mdxJsxTextElement' && + !OPAQUE_ELEMENTS.has(String(child.name ?? '').toLowerCase()) + ) { + collect(child, items); + } else { + items.push(null); + } + } +} + +function directBlocks(node) { + for (const child of node.children ?? []) { + if (!BLOCKS.has(child.type)) { + directBlocks(child); + continue; + } + const items = []; + collect(child, items); + const directed = directQuotes( + items.map((item) => (item === null ? null : typeof item === 'string' ? item : item.value)) + ); + items.forEach((item, i) => { + if (item !== null && typeof item !== 'string') item.value = directed[i]; + }); + } +} + +/** + * Remark plugin: set every quotation mark in a zh-Hans, zh-Hant, ja or ko + * file by its position in its pair. Any other file is returned untouched. + */ +export function remarkCjkQuotes() { + return (tree, file) => { + if (!cjkLocaleOfPath(file?.path ?? file?.history?.[0])) return; + directBlocks(tree); + }; +} diff --git a/scripts/lib/wechat-html.mjs b/scripts/lib/wechat-html.mjs index 8fd3cc3..1e7cd11 100644 --- a/scripts/lib/wechat-html.mjs +++ b/scripts/lib/wechat-html.mjs @@ -5,6 +5,8 @@ // turned into footnote references collected at the end. import { marked } from 'marked'; +import { collapseCjkSegmentBreaks, firstChar, lastChar } from './cjk-segment-break.mjs'; +import { directQuotes, hasCjkLetter } from './cjk-quotes.mjs'; // --- Inline style palette (tweak here to restyle every export) --------------- // Accent: indigo #4f46e5. Designed for WeChat (inline styles only; classes and @@ -39,6 +41,77 @@ const S = { refUrl: 'color:#8c8c8c;font-size:13px;word-break:break-all;', }; +// --- CJK prose in marked's tokens -------------------------------------------- +// The site fixes two things in CJK prose at build time, but the export renders +// with marked, not Astro, so neither step reaches it. Both are applied here to +// marked's inline tokens, with the same definitions the site uses: +// +// * a source line break between two CJK characters is removed +// (`collapseCjkSegmentBreaks`, scripts/lib/cjk-segment-break.mjs), since +// the WeChat reader, like Chromium and Safari, shows it as a space; +// * quotation marks are set by their position in their pair (`directQuotes`, +// scripts/lib/cjk-quotes.mjs). marked leaves ASCII quotes straight; a block +// holding a CJK letter gets the curly marks the site prints, and a block +// without one is left as marked renders it, so an English export is +// unchanged. +// +// Code spans, images, raw HTML and line breaks bound the prose for the first +// (a break next to inline code keeps its space) and are opaque inside it for +// the second (a quotation may wrap inline code). Fenced code is a block token +// with no inline tokens, so neither step ever sees it. + +/** Inline tokens whose children are part of the same sentence. */ +const INLINE_CONTAINERS = new Set(['strong', 'em', 'del', 'link']); + +/** One block's inline content in order: text tokens, and `null` for anything opaque. */ +function inlineItems(tokens, items = []) { + for (const token of tokens) { + if (token.type === 'text' || token.type === 'escape') items.push(token); + else if (INLINE_CONTAINERS.has(token.type) && token.tokens) inlineItems(token.tokens, items); + else items.push(null); + } + return items; +} + +/** The character just outside item `i` of `values`, looking in direction `step`. */ +function neighbourChar(values, i, step) { + for (let j = i + step; j >= 0 && j < values.length; j += step) { + if (values[j] === null) return ''; + if (values[j].length > 0) return step < 0 ? lastChar(values[j]) : firstChar(values[j]); + } + return ''; +} + +function fixInlineRun(tokens) { + const items = inlineItems(tokens); + // Read from the values as lexed, before any edit, as the rehype step does. + const values = items.map((item) => (item ? item.text : null)); + const collapsed = values.map((value, i) => + value !== null && value.includes('\n') + ? collapseCjkSegmentBreaks(value, neighbourChar(values, i, -1), neighbourChar(values, i, 1)) + : value + ); + const prose = collapsed.filter((value) => value !== null).join(''); + const directed = hasCjkLetter(prose) ? directQuotes(collapsed) : collapsed; + items.forEach((item, i) => { + if (item) item.text = directed[i]; + }); +} + +/** + * marked `walkTokens` callback: fixes each token that holds one block's inline + * tokens (a paragraph, a heading, a tight list item's text, a table cell). An + * inline token is visited too and ignored: none of these types is inline, + * except `text`, which holds inline tokens only at block level. + */ +function fixCjkBlock(token) { + if (token.type === 'table') { + for (const cell of [...token.header, ...token.rows.flat()]) fixInlineRun(cell.tokens); + } else if ((token.type === 'paragraph' || token.type === 'heading' || token.type === 'text') && token.tokens) { + fixInlineRun(token.tokens); + } +} + /** Minimal frontmatter splitter — single-line scalars only (title/desc/author). */ function splitFrontmatter(src) { const m = src.match(/^---\n([\s\S]*?)\n---\n?/); @@ -72,7 +145,7 @@ export function toWechatHtml(source) { .join('\n') .trim(); - let html = marked.parse(md, { mangle: false, headerIds: false }); + let html = marked.parse(md, { mangle: false, headerIds: false, walkTokens: fixCjkBlock }); // Code: blocks first (so their gets styled), then inline . html = html.replace(