diff --git a/astro.config.mjs b/astro.config.mjs index f773844..6fe8117 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -4,7 +4,7 @@ import { fileURLToPath } from 'node:url'; import { defineConfig } from 'astro/config'; import mdx from '@astrojs/mdx'; import { rehypeCjkSegmentBreaks } from './scripts/lib/cjk-segment-break.mjs'; -import { remarkCjkQuotes } from './scripts/lib/cjk-quotes.mjs'; +import { remarkQuotes } from './scripts/lib/cjk-quotes.mjs'; import sitemap from '@astrojs/sitemap'; import { readPostLastmods } from './scripts/lib/post-dates.mjs'; @@ -100,11 +100,12 @@ export default defineConfig({ // Chromium and Safari; this removes it from the rendered prose (code // excluded). MDX inherits `markdown` config. See scripts/lib/cjk-segment-break.mjs, // and `pnpm content:lint --dist`, which fails the build on any that remain. - // Smartypants curls quotes by their neighbours, which points the opening mark - // of a zh or ja pair the wrong way; `remarkCjkQuotes` runs after it and sets - // each mark by its position in its pair, in zh-Hans, zh-Hant, ja and ko files - // only. See scripts/lib/cjk-quotes.mjs; `content:lint --dist` guards it too. - markdown: { shikiConfig: { theme: 'github-dark-default' }, remarkPlugins: [remarkCjkQuotes], rehypePlugins: [rehypeCjkSegmentBreaks] }, + // Smartypants curls quotes by their neighbours, and across block boundaries, + // which points marks the wrong way (a zh or ja pair, a heading or table cell + // that opens on a quote, Next ‘26, German „Wort”); `remarkQuotes` runs after it + // and sets each mark by its position in its pair, in every locale. See + // scripts/lib/cjk-quotes.mjs; `content:lint --dist` guards it too. + markdown: { shikiConfig: { theme: 'github-dark-default' }, remarkPlugins: [remarkQuotes], rehypePlugins: [rehypeCjkSegmentBreaks] }, integrations: [ mdx(), sitemap({ diff --git a/scripts/content-lint.mjs b/scripts/content-lint.mjs index e70519a..1a156a9 100644 --- a/scripts/content-lint.mjs +++ b/scripts/content-lint.mjs @@ -20,7 +20,7 @@ import { isCjkWide, lastChar, } from './lib/cjk-segment-break.mjs'; -import { OPAQUE_INLINE, isApostrophe } from './lib/cjk-quotes.mjs'; +import { LOW_OPENERS, OPAQUE_INLINE, isApostrophe } from './lib/cjk-quotes.mjs'; const ROOT = cwd(); const BLOG = path.join(ROOT, 'content', 'blog'); @@ -572,24 +572,28 @@ async function checkBuiltCjkBreaks() { return 1; } -// ─── Built HTML: quotation marks in zh, ja and ko prose point the right way ─ +// ─── Built HTML: quotation marks in prose point the right way ────────────── // // Smartypants curls an ASCII quote by its neighbours, so in Chinese, which puts // no space before a quotation, the opening mark of a pair after a character // comes out closing: 「大多数”给你的业务加上 AI”的说辞」. It also reads a whole // document as one string, so a heading or table cell that starts with a quote -// takes its direction from the block before it (the Korean headings). The -// remark step in astro.config.mjs sets each mark by its position in its pair; -// this reads what the build actually emitted, on every zh, ja and ko page, so -// it also fails when that step is not wired, or when CJK prose reaches a page -// by a road the step never sees (an .astro template, a data file). +// takes its direction from the block before it, in every locale (the Korean +// headings, an English table cell printed ”What exists in our business?”). It +// opens an elided century (Next ‘26), and closes German „Wort" with the English +// ”. The remark step in astro.config.mjs sets each mark by its position in its +// pair; this reads what the build actually emitted, on every page, so it also +// fails when that step is not wired, or when prose reaches a page by a road the +// step never sees (an .astro template, a data file). // // The rule is the step's own, defined once in scripts/lib/cjk-quotes.mjs: in // one block of prose, curly double marks alternate “ ” and end closed, and so -// do single marks, except an apostrophe (`isApostrophe`), which is always ’. -// The HTML scanning below is this gate's own. Straight ASCII marks have no -// direction and are not read; marks inside code, pre, kbd and samp are not -// prose and are not read either. +// do single marks, except an apostrophe (`isApostrophe`), which is always ’. A +// low mark („ ‚) only opens, and its pair closes with “ or ‘ (`LOW_OPENERS`). +// The rule has no per-locale branch, so neither has this gate. The HTML +// scanning below is this gate's own. Straight ASCII marks have no direction +// and are not read; marks inside code, pre, kbd and samp are not prose and are +// not read either. /** Elements that sit inside a sentence without ending its block of prose. */ const QUOTE_INLINE_ELEMENTS = new Set([...TRANSPARENT_ELEMENTS, 'br', 'wbr', 'img']); @@ -673,6 +677,9 @@ function misdirectedQuotes(block) { let marks = 0; let doubleOpen = false; let singleOpen = false; + // The mark that opened the open pair: a low one („ ‚) sets its closer. + let doubleOpener = ''; + let singleOpener = ''; // Where the open pair started, or -1 when its opening mark was already // reported as misdirected (a lone `”` is one problem, not two). let doubleAt = -1; @@ -681,80 +688,122 @@ function misdirectedQuotes(block) { `${problem}: …${block.slice(Math.max(0, at - 14), at)}⟦${block[at] ?? ''}⟧${block.slice(at + 1, at + 15)}…` .replace(/\s+/g, ' ') .replaceAll(OPAQUE_INLINE, '▯'); + const neverClosed = (at) => context(at, `a ${block[at]} never closed in its block`); + const close = (at, opener, fallback) => { + const expected = LOW_OPENERS.get(opener) ?? fallback; + if (block[at] !== expected) { + hits.push(context(at, `a closing mark printed as ${block[at]}${LOW_OPENERS.has(opener) ? `, after ${opener}` : ''}`)); + } + }; for (let i = 0; i < block.length; i++) { const ch = block[i]; - if (ch === '“' || ch === '”') { + if (ch === '„' || ch === '‚') { + // A low mark only opens. One met inside an open pair leaves that pair open. + marks++; + if (ch === '„') { + if (doubleOpen && doubleAt !== -1) hits.push(neverClosed(doubleAt)); + [doubleOpen, doubleOpener, doubleAt] = [true, ch, i]; + } else { + if (singleOpen && singleAt !== -1) hits.push(neverClosed(singleAt)); + [singleOpen, singleOpener, singleAt] = [true, ch, i]; + } + } else if (ch === '“' || ch === '”') { marks++; - const expected = doubleOpen ? '”' : '“'; - if (ch !== expected) hits.push(context(i, doubleOpen ? 'a closing mark printed as “' : 'an opening mark printed as ”')); - doubleOpen = !doubleOpen; - if (doubleOpen) doubleAt = ch === expected ? i : -1; + if (doubleOpen) { + close(i, doubleOpener, '”'); + doubleOpen = false; + } else { + if (ch !== '“') hits.push(context(i, 'an opening mark printed as ”')); + [doubleOpen, doubleOpener, doubleAt] = [true, ch, ch === '“' ? i : -1]; + } } else if (ch === '‘' || ch === '’') { marks++; if (isApostrophe(lastChar(block.slice(0, i)), block.slice(i + 1, i + 6), singleOpen)) { if (ch === '‘') hits.push(context(i, 'an apostrophe printed as ‘')); continue; } - const expected = singleOpen ? '’' : '‘'; - if (ch !== expected) hits.push(context(i, singleOpen ? 'a closing mark printed as ‘' : 'an opening mark printed as ’')); - singleOpen = !singleOpen; - if (singleOpen) singleAt = ch === expected ? i : -1; + if (singleOpen) { + close(i, singleOpener, '’'); + singleOpen = false; + } else { + if (ch !== '‘') hits.push(context(i, 'an opening mark printed as ’')); + [singleOpen, singleOpener, singleAt] = [true, ch, ch === '‘' ? i : -1]; + } } } - if (doubleOpen && doubleAt !== -1) hits.push(context(doubleAt, 'a “ never closed in its block')); - if (singleOpen && singleAt !== -1) hits.push(context(singleAt, 'a ‘ never closed in its block')); + if (doubleOpen && doubleAt !== -1) hits.push(neverClosed(doubleAt)); + if (singleOpen && singleAt !== -1) hits.push(neverClosed(singleAt)); return { marks, hits }; } -/** A page whose document language is Chinese, Japanese or Korean. */ -const CJK_QUOTE_LANG = /]*\blang\s*=\s*["']?(?:zh|ja|ko)(?:-|["'\s>])/i; +/** A page's document language, from ``, for the summary line. */ +const HTML_LANG = /]*\blang\s*=\s*["']?([A-Za-z]+(?:-[A-Za-z0-9]+)*)/i; -async function checkBuiltCjkQuotes() { +async function checkBuiltQuotes() { const pages = (await distFiles(DIST)).filter((file) => file.endsWith('.html')).sort(); const offenders = []; - let cjkPages = 0; + // Pages and curly marks read, per document language, so the summary shows + // which locales were measured. + const langs = new Map(); let marks = 0; let total = 0; for (const page of pages) { const html = await readFile(path.join(DIST, page), 'utf8'); - if (!CJK_QUOTE_LANG.test(html)) continue; - cjkPages++; + const lang = HTML_LANG.exec(html)?.[1] ?? 'no lang'; + const seen = langs.get(lang) ?? { pages: 0, marks: 0 }; + langs.set(lang, seen); + seen.pages++; const hits = []; for (const block of quoteBlocks(html)) { const result = misdirectedQuotes(block); + seen.marks += result.marks; marks += result.marks; hits.push(...result.hits); } total += hits.length; - if (hits.length > 0) offenders.push({ page, hits }); + if (hits.length > 0) offenders.push({ page, lang, hits }); } - if (cjkPages === 0 || marks === 0) { + if (pages.length === 0 || marks === 0) { console.error( - `✗ content lint --dist found ${cjkPages === 0 ? 'no zh, ja or ko page (by ``)' : 'no curly quotation mark on any zh, ja or ko page'} ` + - `among ${pages.length} built page${pages.length === 1 ? '' : 's'} under ${path.relative(ROOT, DIST)}/.\n` + + `✗ content lint --dist found ${pages.length === 0 ? 'no built page' : 'no curly quotation mark on any built page'} ` + + `under ${path.relative(ROOT, DIST)}/ (${pages.length} page${pages.length === 1 ? '' : 's'}).\n` + ` Those are the pages where a quotation mark can point the wrong way, so this checked nothing, ` + `which is not the same as nothing being wrong.` ); return 1; } + const perLang = [...langs] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([lang, seen]) => `${lang} ${seen.pages}/${seen.marks}`) + .join(', '); if (offenders.length === 0) { console.log( - `✓ content lint --dist CJK quotation marks passed (${cjkPages} zh/ja/ko pages, ${marks} curly marks, ` + - `0 pointing the wrong way or left open)` + `✓ content lint --dist quotation marks passed (${pages.length} built pages, ${marks} curly marks, ` + + `0 pointing the wrong way or left open; pages/marks by : ${perLang})` ); return 0; } - for (const { page, hits } of offenders) { + const offendingLangs = new Map(); + for (const { page, lang, hits } of offenders) { + const tally = offendingLangs.get(lang) ?? { pages: 0, hits: 0 }; + offendingLangs.set(lang, tally); + tally.pages++; + tally.hits += hits.length; console.log(`✗ ${page}: ${hits.length} quotation mark${hits.length === 1 ? '' : 's'} pointing the wrong way or left open`); for (const hit of hits.slice(0, 5)) console.log(` ${hit}`); if (hits.length > 5) console.log(` … and ${hits.length - 5} more`); } + const byLang = [...offendingLangs] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([lang, tally]) => `${lang} ${tally.hits} on ${tally.pages}`) + .join(', '); console.error( `\n✗ content lint --dist failed (${total} quotation mark${total === 1 ? '' : 's'} pointing the wrong way or left open on ` + - `${offenders.length} zh/ja/ko page${offenders.length === 1 ? '' : 's'}; ⟦ ⟧ marks each)\n` + - ` In one paragraph, heading or table cell, the first mark of a pair opens and the next closes.\n` + - ` Markdown and MDX in zh-Hans, zh-Hant, ja and ko are set by \`remarkCjkQuotes\` (scripts/lib/cjk-quotes.mjs),\n` + - ` wired in astro.config.mjs \`markdown.remarkPlugins\`: if these pages are posts, that step did not run, or a\n` + + `${offenders.length} page${offenders.length === 1 ? '' : 's'}: ${byLang}; ⟦ ⟧ marks each)\n` + + ` In one paragraph, heading or table cell, the first mark of a pair opens and the next closes; a pair\n` + + ` opened by „ or ‚ closes with “ or ‘, and an apostrophe is always ’.\n` + + ` Markdown and MDX in every locale are set by \`remarkQuotes\` (scripts/lib/cjk-quotes.mjs), wired in\n` + + ` astro.config.mjs \`markdown.remarkPlugins\`: if these pages are posts, that step did not run, or a\n` + ` paragraph holds an odd number of marks (a stray \`"\`, or a quotation that runs over two paragraphs; close it\n` + ` in each). If the text came from somewhere else (an .astro template, a data file), fix the marks in that source.` ); @@ -764,9 +813,9 @@ async function checkBuiltCjkQuotes() { if (args.has('--dist')) { const prose = await checkBuiltProse(); const cjkBreaks = await checkBuiltCjkBreaks(); - const cjkQuotes = await checkBuiltCjkQuotes(); + const quotes = await checkBuiltQuotes(); const links = await checkBuiltLinks(); - exit(prose || cjkBreaks || cjkQuotes || links); + exit(prose || cjkBreaks || quotes || links); } // The frontmatter taxonomy, derived from the module that declares it instead of diff --git a/scripts/lib/cjk-quotes.mjs b/scripts/lib/cjk-quotes.mjs index 375145c..c2e1319 100644 --- a/scripts/lib/cjk-quotes.mjs +++ b/scripts/lib/cjk-quotes.mjs @@ -1,4 +1,7 @@ -// Quotation marks in CJK prose must point the right way. +// Quotation marks in prose must point the right way, in every locale. +// +// (The file keeps its first name: the step began with zh, ja and ko, and the +// WeChat export imports `directQuotes` from here.) // // Astro's default smartypants (remark-smartypants over retext-smartypants) // curls an ASCII quote by looking at the characters around it: after a letter @@ -7,30 +10,37 @@ // that follows a character comes out closing: 「大多数”给你的业务加上 AI”的说辞」. // It also reads the whole document as one string, so a heading or table cell // that starts with a quote takes its direction from the last letter of the -// block before it; that is the Korean case, where words are spaced -// («”다른 AI에게 검토를 맡기면 되지”가 …» as an h2). An audit of the built site -// found 299 such pairs in 16 zh-Hans posts, the same in their zh-Hant copies, -// and 13 headings and table cells in six ko posts. +// block before it. That is the Korean case, where words are spaced +// («”다른 AI에게 검토를 맡기면 되지”가 …» as an h2), and the case in every Latin +// locale too: an English table of questions printed each one as +// ”What exists in our business?”. An apostrophe that elides a century came out +// as an opening mark (Next ‘26). German types „Wort" and smartypants curls the +// ASCII closer as English ”, which German never prints after „: „Wort”. +// Audits of the built site found 299 pairs in 16 zh-Hans posts, the same in +// their zh-Hant copies, 13 headings and table cells in six ko posts, and, once +// the rule ran on every locale, 344 marks on 17 de pages and 22 on 12 +// en/es/fr pages. // // The rule here does not guess from neighbours. Within one block of prose (a // paragraph, a heading, a table cell) double quotation marks come in pairs, so // a mark's direction is its position in the pair: the first opens, the next // closes. A mark the author already typed curly counts the same way, so a // source that mixes `"` and `“ ”` still pairs. Single marks pair the same way, -// except an apostrophe, which is not part of a pair (see `isApostrophe`). +// except an apostrophe, which is not part of a pair (see `isApostrophe`). A +// low mark, „ or ‚, only ever opens, and its pair closes with “ or ‘, the +// German pairs (de is the one locale here that types them; see `LOW_OPENERS`). // // Two consumers share these definitions, so they cannot disagree: // -// * `remarkCjkQuotes`, the build-time fix, wired into -// `markdown.remarkPlugins` in astro.config.mjs (MDX inherits it). It runs -// after smartypants and only on zh-Hans, zh-Hant, ja and ko files, chosen -// by file name, so an English, German, Spanish or French page is not -// touched at all. Every future post in those locales is covered, however -// its quotes were typed. +// * `remarkQuotes`, the build-time fix, wired into `markdown.remarkPlugins` +// in astro.config.mjs (MDX inherits it). It runs after smartypants on +// every Markdown and MDX file, whatever its locale, so every future post is +// covered, however its quotes were typed. The rule has no per-locale +// branch: the German pairs follow from the marks the source typed. // * `pnpm content:lint --dist`, the guard, which reads the BUILT HTML of -// every zh, ja and ko page with its own scanner and fails on a mark whose -// direction disagrees with its position, or a pair left open. It catches -// the step not running, and CJK prose reaching a page by another road. +// every page with its own scanner and fails on a mark whose direction +// disagrees with its position, or a pair left open. It catches the step +// not running, and prose reaching a page by another road. // // The WeChat export (scripts/lib/wechat-html.mjs) renders with marked, which // does not curl quotes at all, and uses `directQuotes` on its own tokens. @@ -39,28 +49,28 @@ // opaque. They sit in the sentence (a quote can wrap inline code), but their // text is neither edited nor read. // -// The marks stay “ ” and ‘ ’. AGENTS.md records no zh quotation style, and the -// sources already type “ ” (354 pairs in zh-Hans) far more than 「」 (23), so -// the author's choice of mark is kept and only the direction is corrected; -// a source that types 「」 is left exactly as written. +// Only the direction is corrected; the author's choice of mark is kept. +// AGENTS.md records no quotation style for any locale. The zh sources type “ ” +// (354 pairs in zh-Hans) far more than 「」 (23), so the marks there stay “ ” +// and ‘ ’; a source that types 「」, « » or „ “ keeps it, and an ASCII pair +// becomes “ ” in every locale, as smartypants prints it. import { OPAQUE_ELEMENTS } from './cjk-segment-break.mjs'; -/** Locales whose prose puts no space before a quotation, or whose files this step owns. */ -export const CJK_QUOTE_LOCALES = new Set(['zh-Hans', 'zh-Hant', 'ja', 'ko']); +export const DOUBLE_QUOTES = new Set(['"', '“', '”', '„']); +export const SINGLE_QUOTES = new Set(["'", '‘', '’', '‚']); /** - * The CJK locale of a content file from its name, or null. A translation is - * `index..mdx` (or flat `..mdx`); the English original - * carries no locale and is never matched. + * A low mark and the mark that closes its pair. „ and ‚ never close a pair, + * and never stand for an apostrophe: they open, and the pair they open closes + * with “ or ‘ (German „Wort“, ‚Wort‘), whatever the source typed for the + * closer. A pair opened any other way closes with ” or ’. Smartypants turns + * the ASCII closer of `„Wort"` into ”, a closing mark German never prints. */ -export function cjkLocaleOfPath(file) { - const m = /\.([A-Za-z]{2,3}(?:-[A-Za-z]{2,4})?)\.mdx?$/.exec(String(file ?? '')); - return m && CJK_QUOTE_LOCALES.has(m[1]) ? m[1] : null; -} - -export const DOUBLE_QUOTES = new Set(['"', '“', '”']); -export const SINGLE_QUOTES = new Set(["'", '‘', '’']); +export const LOW_OPENERS = new Map([ + ['„', '“'], + ['‚', '‘'], +]); /** * Stands for an opaque inline (inline code, an image) in a block's text. It @@ -97,12 +107,15 @@ function lastCharOf(text) { * Directs every quotation mark in one block of prose. `segments` is the * block's text in document order: a string is prose, `null` an opaque inline. * Returns the segments with each double and single mark set by its position - * in its pair; nothing else changes, and `null` stays `null`. + * in its pair; nothing else changes, and `null` stays `null`. A low mark keeps + * its shape and sets the closer of its pair (`LOW_OPENERS`). */ export function directQuotes(segments) { const full = segments.map((s) => (s === null ? OPAQUE_INLINE : s)).join(''); let doubleOpen = false; + let doubleCloser = '”'; let singleOpen = false; + let singleCloser = '’'; let at = 0; return segments.map((segment) => { if (segment === null) { @@ -112,15 +125,29 @@ export function directQuotes(segments) { let out = ''; for (const ch of segment) { if (DOUBLE_QUOTES.has(ch)) { - out += doubleOpen ? '”' : '“'; - doubleOpen = !doubleOpen; + if (doubleOpen && !LOW_OPENERS.has(ch)) { + out += doubleCloser; + doubleOpen = false; + } else { + out += LOW_OPENERS.has(ch) ? ch : '“'; + doubleCloser = LOW_OPENERS.get(ch) ?? '”'; + doubleOpen = true; + } + } else if (LOW_OPENERS.has(ch)) { + out += ch; + singleCloser = LOW_OPENERS.get(ch); + singleOpen = true; } else if (SINGLE_QUOTES.has(ch)) { const before = lastCharOf(full.slice(Math.max(0, at - 2), at)); if (isApostrophe(before, full.slice(at + 1, at + 6), singleOpen)) { out += '’'; + } else if (singleOpen) { + out += singleCloser; + singleOpen = false; } else { - out += singleOpen ? '’' : '‘'; - singleOpen = !singleOpen; + out += '‘'; + singleCloser = '’'; + singleOpen = true; } } else { out += ch; @@ -186,12 +213,11 @@ function directBlocks(node) { } /** - * Remark plugin: set every quotation mark in a zh-Hans, zh-Hant, ja or ko - * file by its position in its pair. Any other file is returned untouched. + * Remark plugin: set every quotation mark in a Markdown or MDX file by its + * position in its pair, whatever the file's locale. */ -export function remarkCjkQuotes() { - return (tree, file) => { - if (!cjkLocaleOfPath(file?.path ?? file?.history?.[0])) return; +export function remarkQuotes() { + return (tree) => { directBlocks(tree); }; }