fix(blog): remove the space a hard-wrapped line leaves between two CJK characters

astro.config.mjs

@@ -3,6 +3,7 @@ import { readdir, rename, rmdir } from 'node:fs/promises';
 import { fileURLToPath } from 'node:url';
 import { defineConfig } from 'astro/config';
 import mdx from '@astrojs/mdx';
+import { rehypeCjkSegmentBreaks } from './scripts/lib/cjk-segment-break.mjs';
 import sitemap from '@astrojs/sitemap';
 import { readPostLastmods } from './scripts/lib/post-dates.mjs';
 
@@ -94,7 +95,11 @@ export default defineConfig({
   // `github-dark-default` keeps the block dark (BlogPost.astro draws the
   // code-block scrollbar light for it) and puts comments at #8b949e on
   // #0d1117, 6.15:1, the lowest of the token colours the posts use.
-  markdown: { shikiConfig: { theme: 'github-dark-default' } },
+  // A source line break between two CJK characters renders as a stray space in
+  // Chromium and Safari; this removes it from the rendered prose (code
+  // excluded). MDX inherits `markdown` config. See scripts/lib/cjk-segment-break.mjs,
+  // and `pnpm content:lint --dist`, which fails the build on any that remain.
+  markdown: { shikiConfig: { theme: 'github-dark-default' }, rehypePlugins: [rehypeCjkSegmentBreaks] },
   integrations: [
     mdx(),
     sitemap({

scripts/content-lint.mjs

@@ -12,6 +12,14 @@ import { pathToFileURL } from 'node:url';
 import { cwd, argv, exit } from 'node:process';
 import yaml from 'js-yaml';
 import { splitFrontmatter as splitFence } from './lib/frontmatter.mjs';
+import {
+  OPAQUE_ELEMENTS,
+  SEGMENT_BREAK,
+  TRANSPARENT_ELEMENTS,
+  firstChar,
+  isCjkWide,
+  lastChar,
+} from './lib/cjk-segment-break.mjs';
 
 const ROOT = cwd();
 const BLOG = path.join(ROOT, 'content', 'blog');
@@ -423,10 +431,151 @@ async function checkBuiltLinks() {
   return 1;
 }
 
+// ─── Built HTML: no segment break left between two CJK characters ─────────
+//
+// A hard-wrapped CJK source puts a newline between two characters of one
+// sentence, and the HTML carries it as a segment break. CSS Text 3 removes such
+// a break between two wide characters, but Chromium and Safari render it as a
+// space, so the reader sees 「数据 也已经」 mid-sentence. The rehype step in
+// astro.config.mjs removes these from everything Markdown/MDX renders; this
+// reads what the build actually emitted, so it also fails when that step is not
+// wired, or when CJK prose reaches a page by a road the step never sees (an
+// .astro template, a data file, a new renderer).
+//
+// Which characters count, which elements are prose and which are opaque is
+// defined once in scripts/lib/cjk-segment-break.mjs and shared with the fix;
+// the HTML scanning below is this gate's own. A break next to code, between
+// CJK and Latin, after an ambiguous-width `——`, or between two Hangul words is
+// a real space and is not flagged.
+
+/**
+ * Runs of inline prose: text joined across transparent phrasing elements
+ * (`<strong>`, `<a>`, …) and cut at every other tag, with opaque elements
+ * (code, pre, scripts, …) dropped entirely, as the rehype step sees them.
+ */
+function proseRuns(html) {
+  const runs = [];
+  let run = '';
+  let opaqueDepth = 0;
+  const cut = () => {
+    if (run) runs.push(run);
+    run = '';
+  };
+  let i = 0;
+  while (i < html.length) {
+    const lt = html.indexOf('<', i);
+    const stop = lt === -1 ? html.length : lt;
+    if (stop > i && opaqueDepth === 0) run += decodeEntities(html.slice(i, stop));
+    if (lt === -1) break;
+    if (html.startsWith('<!--', lt)) {
+      cut();
+      const end = html.indexOf('-->', lt + 4);
+      i = end === -1 ? html.length : end + 3;
+      continue;
+    }
+    if (html.startsWith('<!', lt)) {
+      cut();
+      const end = tagEnd(html, lt + 2);
+      i = end === -1 ? html.length : end + 1;
+      continue;
+    }
+    const tag = /^<(\/?)([a-zA-Z][a-zA-Z0-9-]*)/.exec(html.slice(lt, lt + 64));
+    if (!tag) {
+      if (opaqueDepth === 0) run += '<';
+      i = lt + 1;
+      continue;
+    }
+    const closing = tag[1] === '/';
+    const name = tag[2].toLowerCase();
+    const end = tagEnd(html, lt + tag[0].length);
+    if (end === -1) break;
+    const selfClosing = html[end - 1] === '/';
+    i = end + 1;
+    if (!closing && RAW_TEXT_ELEMENTS.has(name)) {
+      // Skip the body and its closing tag, so the closing tag is never
+      // mistaken for the end of an enclosing opaque element.
+      cut();
+      const close = html.toLowerCase().indexOf(`</${name}`, i);
+      const closeEnd = close === -1 ? -1 : tagEnd(html, close + name.length + 2);
+      i = closeEnd === -1 ? html.length : closeEnd + 1;
+      continue;
+    }
+    if (OPAQUE_ELEMENTS.has(name)) {
+      cut();
+      if (closing) opaqueDepth = Math.max(0, opaqueDepth - 1);
+      else if (!selfClosing) opaqueDepth++;
+      continue;
+    }
+    // A transparent element does not end the run it sits in; any other does.
+    if (!TRANSPARENT_ELEMENTS.has(name)) cut();
+  }
+  cut();
+  return runs;
+}
+
+/**
+ * A page whose document language is Chinese or Japanese: where a CJK line
+ * break would show. Counted so a build without them fails as unmeasured (every
+ * page carries CJK in its language menu, so "has CJK text" would not do).
+ */
+const CJK_LANG = /<html\b[^>]*\blang\s*=\s*["']?(?:zh|ja)(?:-|["'\s>])/i;
+
+async function checkBuiltCjkBreaks() {
+  const pages = (await distFiles(DIST)).filter((file) => file.endsWith('.html')).sort();
+  const offenders = [];
+  let cjkPages = 0;
+  let total = 0;
+  for (const page of pages) {
+    const html = await readFile(path.join(DIST, page), 'utf8');
+    if (CJK_LANG.test(html)) cjkPages++;
+    const hits = [];
+    for (const run of proseRuns(html)) {
+      for (const m of run.matchAll(SEGMENT_BREAK)) {
+        const end = m.index + m[0].length;
+        if (!isCjkWide(lastChar(run.slice(0, m.index))) || !isCjkWide(firstChar(run.slice(end)))) continue;
+        hits.push(`${run.slice(Math.max(0, m.index - 16), m.index)}⏎${run.slice(end, end + 16)}`.replace(/\s+/g, ' '));
+      }
+    }
+    total += hits.length;
+    if (hits.length > 0) offenders.push({ page, hits });
+  }
+  if (cjkPages === 0) {
+    console.error(
+      `✗ content lint --dist found no zh or ja page (by \`<html lang>\`) among ${pages.length} built ` +
+        `page${pages.length === 1 ? '' : 's'} under ${path.relative(ROOT, DIST)}/.\n` +
+        `  The zh-Hans, zh-Hant and ja pages are where a CJK line break would show, so a build without them ` +
+        `checked nothing, which is not the same as nothing being wrong.`
+    );
+    return 1;
+  }
+  if (offenders.length === 0) {
+    console.log(
+      `✓ content lint --dist CJK line breaks passed (${pages.length} built pages, ${cjkPages} in zh/ja, ` +
+        `0 segment breaks between CJK characters)`
+    );
+    return 0;
+  }
+  for (const { page, hits } of offenders) {
+    console.log(`✗ ${page}: ${hits.length} line break${hits.length === 1 ? '' : 's'} between CJK characters in rendered prose`);
+    for (const hit of hits.slice(0, 5)) console.log(`    …${hit}…`);
+    if (hits.length > 5) console.log(`    … and ${hits.length - 5} more`);
+  }
+  console.error(
+    `\n✗ content lint --dist failed (${total} segment break${total === 1 ? '' : 's'} between CJK characters on ` +
+      `${offenders.length} page${offenders.length === 1 ? '' : 's'}; ⏎ marks each)\n` +
+      `  Chromium and Safari render a source line break between two CJK characters as a space mid-sentence.\n` +
+      `  Markdown and MDX are fixed by \`rehypeCjkSegmentBreaks\` (scripts/lib/cjk-segment-break.mjs), wired in\n` +
+      `  astro.config.mjs \`markdown.rehypePlugins\`: if these pages are posts, that step did not run. If the text\n` +
+      `  came from somewhere else (an .astro template, a data file), join the line in that source.`
+  );
+  return 1;
+}
+
 if (args.has('--dist')) {
   const prose = await checkBuiltProse();
+  const cjkBreaks = await checkBuiltCjkBreaks();
   const links = await checkBuiltLinks();
-  exit(prose || links);
+  exit(prose || cjkBreaks || links);
 }
 
 // The frontmatter taxonomy, derived from the module that declares it instead of

scripts/lib/cjk-segment-break.mjs

@@ -0,0 +1,182 @@
+// A source line break between two CJK characters must not reach a reader as a
+// space.
+//
+// Markdown keeps a soft line break in a paragraph as a newline in the text, and
+// the HTML carries it through as a segment break. CSS Text 3 says a segment
+// break between two East Asian wide characters is removed, but Chromium and
+// Safari do not implement that yet: they render it as a space. A hard-wrapped
+// zh-Hans source therefore shows a stray space mid-sentence (「已经在运转，数据
+// 也已经在那里。」), in zh-Hans and in the zh-Hant generated from it. An audit of
+// the built site found 202 of them in four posts and their zh-Hant copies.
+//
+// Two consumers share the definitions in this file, so they cannot disagree:
+//
+//   * `rehypeCjkSegmentBreaks`, the build-time fix, wired into
+//     `markdown.rehypePlugins` in astro.config.mjs (MDX inherits it). It removes
+//     the break in the rendered tree, so a hard-wrapped source stays legal and
+//     the fix protects every future post in every CJK locale, not only the four
+//     posts reflowed by hand.
+//   * `pnpm content:lint --dist`, the guard, which reads the BUILT HTML with its
+//     own scanner and fails on any segment break left between two of these
+//     characters in prose. It catches the plugin not running, and a producer the
+//     plugin never sees (an .astro template, a new render path).
+//
+// What is removed is exactly the CSS Text 3 rule (§4.1.3, segment break
+// transformation), not a Chinese-specific heuristic:
+//
+//   * Both neighbours must be East Asian Width Fullwidth, Wide or Halfwidth.
+//     Ambiguous characters do not count — `——`, `…`, `“ ”` and `·` sit in
+//     Latin text too, and the posts set ` —— ` with a space on both sides, so
+//     a break after `——` is that space.
+//   * Neither neighbour may be Hangul. Korean separates words with spaces, so a
+//     line break between two Hangul words is a real word space.
+//   * A break between CJK and Latin (`支持\nAI`) stays a space: that is the
+//     spacing these posts use between the two scripts anyway.
+//
+// Code is never touched: `pre`, `code`, `kbd` and `samp` are opaque. Their text
+// is neither edited nor used as the neighbour of a break outside them, so a
+// break next to inline code keeps its space.
+
+/**
+ * Code point ranges whose East Asian Width is Wide, Fullwidth or Halfwidth.
+ * Hangul inside them is excluded by `isCjkWide`, not by the ranges.
+ */
+const WIDE_RANGES = [
+  [0x2e80, 0x303e], // CJK radicals, Kangxi radicals, ideographic description, CJK symbols and punctuation (U+303F is narrow)
+  [0x3041, 0x33ff], // kana, bopomofo, kanbun, CJK strokes, enclosed CJK, CJK compatibility
+  [0x3400, 0x4dbf], // CJK unified ideographs extension A (U+4DC0..U+4DFF hexagrams are narrow)
+  [0x4e00, 0x9fff], // CJK unified ideographs
+  [0xf900, 0xfaff], // CJK compatibility ideographs
+  [0xfe10, 0xfe19], // vertical forms
+  [0xfe30, 0xfe6f], // CJK compatibility forms, small form variants
+  [0xff01, 0xffee], // fullwidth and halfwidth forms (，。：（）！？ and halfwidth kana)
+  [0x16fe0, 0x18aff], // ideographic symbols, Tangut
+  [0x1b000, 0x1b2ff], // kana supplement and extensions, Nushu
+  [0x20000, 0x3fffd], // CJK unified ideographs extensions B and later
+];
+
+const HANGUL = /\p{Script=Hangul}/u;
+
+/** True when the CSS segment-break rule removes a break next to `ch`. */
+export function isCjkWide(ch) {
+  if (!ch) return false;
+  const cp = ch.codePointAt(0);
+  let inRange = false;
+  for (const [lo, hi] of WIDE_RANGES) {
+    if (cp < lo) break;
+    if (cp <= hi) {
+      inRange = true;
+      break;
+    }
+  }
+  return inRange && !HANGUL.test(ch);
+}
+
+/**
+ * One segment break with the collapsible spaces and tabs around it. CSS removes
+ * those spaces before it transforms the break, so the run is one unit.
+ */
+export const SEGMENT_BREAK = /[ \t]*\n[ \t\n]*/g;
+
+/**
+ * Phrasing elements that do not interrupt the prose around them: a break just
+ * outside `<strong>…</strong>` still sits between the character inside and the
+ * one outside. Any element not listed here (and not opaque) bounds the prose.
+ */
+export const TRANSPARENT_ELEMENTS = new Set([
+  'a', 'abbr', 'b', 'bdi', 'bdo', 'cite', 'data', 'del', 'dfn', 'em', 'i', 'ins',
+  'mark', 's', 'small', 'span', 'strong', 'sub', 'sup', 'time', 'u', 'var',
+]);
+
+/** Elements whose text is not prose: never edited, never a neighbour. */
+export const OPAQUE_ELEMENTS = new Set([
+  'code', 'kbd', 'pre', 'samp', 'script', 'style', 'textarea', 'title', 'math', 'svg',
+]);
+
+export function lastChar(text) {
+  if (!text) return '';
+  const low = text.charCodeAt(text.length - 1);
+  if (low >= 0xdc00 && low <= 0xdfff && text.length > 1) {
+    const high = text.charCodeAt(text.length - 2);
+    if (high >= 0xd800 && high <= 0xdbff) return text.slice(-2);
+  }
+  return text.slice(-1);
+}
+
+export function firstChar(text) {
+  if (!text) return '';
+  return String.fromCodePoint(text.codePointAt(0));
+}
+
+/**
+ * Removes every segment break in `value` whose neighbours are both CJK wide.
+ * `before`/`after` are the characters just outside `value` (empty at a
+ * boundary), used when a break sits at its very start or end.
+ */
+export function collapseCjkSegmentBreaks(value, before = '', after = '') {
+  return value.replace(SEGMENT_BREAK, (run, offset) => {
+    const prev = offset > 0 ? lastChar(value.slice(0, offset)) : before;
+    const end = offset + run.length;
+    const next = end < value.length ? firstChar(value.slice(end)) : after;
+    return isCjkWide(prev) && isCjkWide(next) ? '' : run;
+  });
+}
+
+/**
+ * Text nodes of a hast tree in document order, with `null` wherever the prose
+ * is interrupted (a block, a line break, an image, code, raw HTML, an MDX
+ * component or expression — anything whose rendering is not plain inline text).
+ */
+function flatten(node, items) {
+  for (const child of node.children ?? []) {
+    if (child.type === 'text') {
+      items.push(child);
+    } else if (child.type === 'element') {
+      const name = child.tagName.toLowerCase();
+      if (OPAQUE_ELEMENTS.has(name)) {
+        items.push(null);
+      } else if (TRANSPARENT_ELEMENTS.has(name)) {
+        flatten(child, items);
+      } else {
+        items.push(null);
+        flatten(child, items);
+        items.push(null);
+      }
+    } else {
+      items.push(null);
+      if (child.children) {
+        flatten(child, items);
+        items.push(null);
+      }
+    }
+  }
+}
+
+/** The character just outside item `i`, looking in direction `step`. */
+function neighbour(values, i, step) {
+  for (let j = i + step; j >= 0 && j < values.length; j += step) {
+    const value = values[j];
+    if (value === null) return '';
+    if (value.length > 0) return step < 0 ? lastChar(value) : firstChar(value);
+  }
+  return '';
+}
+
+/** Rehype plugin: drop segment breaks between two CJK wide characters in prose. */
+export function rehypeCjkSegmentBreaks() {
+  return (tree) => {
+    const items = [];
+    flatten(tree, items);
+    // Neighbours are read from the values as parsed, before any edit, so the
+    // result does not depend on the order the nodes are visited in.
+    const values = items.map((item) => (item ? item.value : null));
+    items.forEach((item, i) => {
+      if (!item || !item.value.includes('\n')) return;
+      item.value = collapseCjkSegmentBreaks(
+        values[i],
+        neighbour(values, i, -1),
+        neighbour(values, i, 1)
+      );
+    });
+  };
+}