diff --git a/astro.config.mjs b/astro.config.mjs index ac28d54..17e8364 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -3,6 +3,7 @@ import { readdir, rename, rmdir } from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; import { defineConfig } from 'astro/config'; import mdx from '@astrojs/mdx'; +import { rehypeCjkSegmentBreaks } from './scripts/lib/cjk-segment-break.mjs'; import sitemap from '@astrojs/sitemap'; import { readPostLastmods } from './scripts/lib/post-dates.mjs'; @@ -94,7 +95,11 @@ export default defineConfig({ // `github-dark-default` keeps the block dark (BlogPost.astro draws the // code-block scrollbar light for it) and puts comments at #8b949e on // #0d1117, 6.15:1, the lowest of the token colours the posts use. - markdown: { shikiConfig: { theme: 'github-dark-default' } }, + // A source line break between two CJK characters renders as a stray space in + // Chromium and Safari; this removes it from the rendered prose (code + // excluded). MDX inherits `markdown` config. See scripts/lib/cjk-segment-break.mjs, + // and `pnpm content:lint --dist`, which fails the build on any that remain. + markdown: { shikiConfig: { theme: 'github-dark-default' }, rehypePlugins: [rehypeCjkSegmentBreaks] }, integrations: [ mdx(), sitemap({ diff --git a/scripts/content-lint.mjs b/scripts/content-lint.mjs index 6392a33..a1c150b 100644 --- a/scripts/content-lint.mjs +++ b/scripts/content-lint.mjs @@ -12,6 +12,14 @@ import { pathToFileURL } from 'node:url'; import { cwd, argv, exit } from 'node:process'; import yaml from 'js-yaml'; import { splitFrontmatter as splitFence } from './lib/frontmatter.mjs'; +import { + OPAQUE_ELEMENTS, + SEGMENT_BREAK, + TRANSPARENT_ELEMENTS, + firstChar, + isCjkWide, + lastChar, +} from './lib/cjk-segment-break.mjs'; const ROOT = cwd(); const BLOG = path.join(ROOT, 'content', 'blog'); @@ -423,10 +431,151 @@ async function checkBuiltLinks() { return 1; } +// ─── Built HTML: no segment break left between two CJK characters ───────── +// +// A hard-wrapped CJK source puts a newline between two characters of one +// sentence, and the HTML carries it as a segment break. CSS Text 3 removes such +// a break between two wide characters, but Chromium and Safari render it as a +// space, so the reader sees 「数据 也已经」 mid-sentence. The rehype step in +// astro.config.mjs removes these from everything Markdown/MDX renders; this +// reads what the build actually emitted, so it also fails when that step is not +// wired, or when CJK prose reaches a page by a road the step never sees (an +// .astro template, a data file, a new renderer). +// +// Which characters count, which elements are prose and which are opaque is +// defined once in scripts/lib/cjk-segment-break.mjs and shared with the fix; +// the HTML scanning below is this gate's own. A break next to code, between +// CJK and Latin, after an ambiguous-width `——`, or between two Hangul words is +// a real space and is not flagged. + +/** + * Runs of inline prose: text joined across transparent phrasing elements + * (``, ``, …) and cut at every other tag, with opaque elements + * (code, pre, scripts, …) dropped entirely, as the rehype step sees them. + */ +function proseRuns(html) { + const runs = []; + let run = ''; + let opaqueDepth = 0; + const cut = () => { + if (run) runs.push(run); + run = ''; + }; + let i = 0; + while (i < html.length) { + const lt = html.indexOf('<', i); + const stop = lt === -1 ? html.length : lt; + if (stop > i && opaqueDepth === 0) run += decodeEntities(html.slice(i, stop)); + if (lt === -1) break; + if (html.startsWith('', lt + 4); + i = end === -1 ? html.length : end + 3; + continue; + } + if (html.startsWith(']*\blang\s*=\s*["']?(?:zh|ja)(?:-|["'\s>])/i; + +async function checkBuiltCjkBreaks() { + const pages = (await distFiles(DIST)).filter((file) => file.endsWith('.html')).sort(); + const offenders = []; + let cjkPages = 0; + let total = 0; + for (const page of pages) { + const html = await readFile(path.join(DIST, page), 'utf8'); + if (CJK_LANG.test(html)) cjkPages++; + const hits = []; + for (const run of proseRuns(html)) { + for (const m of run.matchAll(SEGMENT_BREAK)) { + const end = m.index + m[0].length; + if (!isCjkWide(lastChar(run.slice(0, m.index))) || !isCjkWide(firstChar(run.slice(end)))) continue; + hits.push(`${run.slice(Math.max(0, m.index - 16), m.index)}⏎${run.slice(end, end + 16)}`.replace(/\s+/g, ' ')); + } + } + total += hits.length; + if (hits.length > 0) offenders.push({ page, hits }); + } + if (cjkPages === 0) { + console.error( + `✗ content lint --dist found no zh or ja page (by \`\`) among ${pages.length} built ` + + `page${pages.length === 1 ? '' : 's'} under ${path.relative(ROOT, DIST)}/.\n` + + ` The zh-Hans, zh-Hant and ja pages are where a CJK line break would show, so a build without them ` + + `checked nothing, which is not the same as nothing being wrong.` + ); + return 1; + } + if (offenders.length === 0) { + console.log( + `✓ content lint --dist CJK line breaks passed (${pages.length} built pages, ${cjkPages} in zh/ja, ` + + `0 segment breaks between CJK characters)` + ); + return 0; + } + for (const { page, hits } of offenders) { + console.log(`✗ ${page}: ${hits.length} line break${hits.length === 1 ? '' : 's'} between CJK characters in rendered prose`); + for (const hit of hits.slice(0, 5)) console.log(` …${hit}…`); + if (hits.length > 5) console.log(` … and ${hits.length - 5} more`); + } + console.error( + `\n✗ content lint --dist failed (${total} segment break${total === 1 ? '' : 's'} between CJK characters on ` + + `${offenders.length} page${offenders.length === 1 ? '' : 's'}; ⏎ marks each)\n` + + ` Chromium and Safari render a source line break between two CJK characters as a space mid-sentence.\n` + + ` Markdown and MDX are fixed by \`rehypeCjkSegmentBreaks\` (scripts/lib/cjk-segment-break.mjs), wired in\n` + + ` astro.config.mjs \`markdown.rehypePlugins\`: if these pages are posts, that step did not run. If the text\n` + + ` came from somewhere else (an .astro template, a data file), join the line in that source.` + ); + return 1; +} + if (args.has('--dist')) { const prose = await checkBuiltProse(); + const cjkBreaks = await checkBuiltCjkBreaks(); const links = await checkBuiltLinks(); - exit(prose || links); + exit(prose || cjkBreaks || links); } // The frontmatter taxonomy, derived from the module that declares it instead of diff --git a/scripts/lib/cjk-segment-break.mjs b/scripts/lib/cjk-segment-break.mjs new file mode 100644 index 0000000..053be23 --- /dev/null +++ b/scripts/lib/cjk-segment-break.mjs @@ -0,0 +1,182 @@ +// A source line break between two CJK characters must not reach a reader as a +// space. +// +// Markdown keeps a soft line break in a paragraph as a newline in the text, and +// the HTML carries it through as a segment break. CSS Text 3 says a segment +// break between two East Asian wide characters is removed, but Chromium and +// Safari do not implement that yet: they render it as a space. A hard-wrapped +// zh-Hans source therefore shows a stray space mid-sentence (「已经在运转，数据 +// 也已经在那里。」), in zh-Hans and in the zh-Hant generated from it. An audit of +// the built site found 202 of them in four posts and their zh-Hant copies. +// +// Two consumers share the definitions in this file, so they cannot disagree: +// +// * `rehypeCjkSegmentBreaks`, the build-time fix, wired into +// `markdown.rehypePlugins` in astro.config.mjs (MDX inherits it). It removes +// the break in the rendered tree, so a hard-wrapped source stays legal and +// the fix protects every future post in every CJK locale, not only the four +// posts reflowed by hand. +// * `pnpm content:lint --dist`, the guard, which reads the BUILT HTML with its +// own scanner and fails on any segment break left between two of these +// characters in prose. It catches the plugin not running, and a producer the +// plugin never sees (an .astro template, a new render path). +// +// What is removed is exactly the CSS Text 3 rule (§4.1.3, segment break +// transformation), not a Chinese-specific heuristic: +// +// * Both neighbours must be East Asian Width Fullwidth, Wide or Halfwidth. +// Ambiguous characters do not count — `——`, `…`, `“ ”` and `·` sit in +// Latin text too, and the posts set ` —— ` with a space on both sides, so +// a break after `——` is that space. +// * Neither neighbour may be Hangul. Korean separates words with spaces, so a +// line break between two Hangul words is a real word space. +// * A break between CJK and Latin (`支持\nAI`) stays a space: that is the +// spacing these posts use between the two scripts anyway. +// +// Code is never touched: `pre`, `code`, `kbd` and `samp` are opaque. Their text +// is neither edited nor used as the neighbour of a break outside them, so a +// break next to inline code keeps its space. + +/** + * Code point ranges whose East Asian Width is Wide, Fullwidth or Halfwidth. + * Hangul inside them is excluded by `isCjkWide`, not by the ranges. + */ +const WIDE_RANGES = [ + [0x2e80, 0x303e], // CJK radicals, Kangxi radicals, ideographic description, CJK symbols and punctuation (U+303F is narrow) + [0x3041, 0x33ff], // kana, bopomofo, kanbun, CJK strokes, enclosed CJK, CJK compatibility + [0x3400, 0x4dbf], // CJK unified ideographs extension A (U+4DC0..U+4DFF hexagrams are narrow) + [0x4e00, 0x9fff], // CJK unified ideographs + [0xf900, 0xfaff], // CJK compatibility ideographs + [0xfe10, 0xfe19], // vertical forms + [0xfe30, 0xfe6f], // CJK compatibility forms, small form variants + [0xff01, 0xffee], // fullwidth and halfwidth forms (，。：（）！？ and halfwidth kana) + [0x16fe0, 0x18aff], // ideographic symbols, Tangut + [0x1b000, 0x1b2ff], // kana supplement and extensions, Nushu + [0x20000, 0x3fffd], // CJK unified ideographs extensions B and later +]; + +const HANGUL = /\p{Script=Hangul}/u; + +/** True when the CSS segment-break rule removes a break next to `ch`. */ +export function isCjkWide(ch) { + if (!ch) return false; + const cp = ch.codePointAt(0); + let inRange = false; + for (const [lo, hi] of WIDE_RANGES) { + if (cp < lo) break; + if (cp <= hi) { + inRange = true; + break; + } + } + return inRange && !HANGUL.test(ch); +} + +/** + * One segment break with the collapsible spaces and tabs around it. CSS removes + * those spaces before it transforms the break, so the run is one unit. + */ +export const SEGMENT_BREAK = /[ \t]*\n[ \t\n]*/g; + +/** + * Phrasing elements that do not interrupt the prose around them: a break just + * outside `…` still sits between the character inside and the + * one outside. Any element not listed here (and not opaque) bounds the prose. + */ +export const TRANSPARENT_ELEMENTS = new Set([ + 'a', 'abbr', 'b', 'bdi', 'bdo', 'cite', 'data', 'del', 'dfn', 'em', 'i', 'ins', + 'mark', 's', 'small', 'span', 'strong', 'sub', 'sup', 'time', 'u', 'var', +]); + +/** Elements whose text is not prose: never edited, never a neighbour. */ +export const OPAQUE_ELEMENTS = new Set([ + 'code', 'kbd', 'pre', 'samp', 'script', 'style', 'textarea', 'title', 'math', 'svg', +]); + +export function lastChar(text) { + if (!text) return ''; + const low = text.charCodeAt(text.length - 1); + if (low >= 0xdc00 && low <= 0xdfff && text.length > 1) { + const high = text.charCodeAt(text.length - 2); + if (high >= 0xd800 && high <= 0xdbff) return text.slice(-2); + } + return text.slice(-1); +} + +export function firstChar(text) { + if (!text) return ''; + return String.fromCodePoint(text.codePointAt(0)); +} + +/** + * Removes every segment break in `value` whose neighbours are both CJK wide. + * `before`/`after` are the characters just outside `value` (empty at a + * boundary), used when a break sits at its very start or end. + */ +export function collapseCjkSegmentBreaks(value, before = '', after = '') { + return value.replace(SEGMENT_BREAK, (run, offset) => { + const prev = offset > 0 ? lastChar(value.slice(0, offset)) : before; + const end = offset + run.length; + const next = end < value.length ? firstChar(value.slice(end)) : after; + return isCjkWide(prev) && isCjkWide(next) ? '' : run; + }); +} + +/** + * Text nodes of a hast tree in document order, with `null` wherever the prose + * is interrupted (a block, a line break, an image, code, raw HTML, an MDX + * component or expression — anything whose rendering is not plain inline text). + */ +function flatten(node, items) { + for (const child of node.children ?? []) { + if (child.type === 'text') { + items.push(child); + } else if (child.type === 'element') { + const name = child.tagName.toLowerCase(); + if (OPAQUE_ELEMENTS.has(name)) { + items.push(null); + } else if (TRANSPARENT_ELEMENTS.has(name)) { + flatten(child, items); + } else { + items.push(null); + flatten(child, items); + items.push(null); + } + } else { + items.push(null); + if (child.children) { + flatten(child, items); + items.push(null); + } + } + } +} + +/** The character just outside item `i`, looking in direction `step`. */ +function neighbour(values, i, step) { + for (let j = i + step; j >= 0 && j < values.length; j += step) { + const value = values[j]; + if (value === null) return ''; + if (value.length > 0) return step < 0 ? lastChar(value) : firstChar(value); + } + return ''; +} + +/** Rehype plugin: drop segment breaks between two CJK wide characters in prose. */ +export function rehypeCjkSegmentBreaks() { + return (tree) => { + const items = []; + flatten(tree, items); + // Neighbours are read from the values as parsed, before any edit, so the + // result does not depend on the order the nodes are visited in. + const values = items.map((item) => (item ? item.value : null)); + items.forEach((item, i) => { + if (!item || !item.value.includes('\n')) return; + item.value = collapseCjkSegmentBreaks( + values[i], + neighbour(values, i, -1), + neighbour(values, i, 1) + ); + }); + }; +}