refactor(sync): route mdx-to-markdown through the shared fence tracker

changelog.d/fixed-569-mdx-shared-fence-rule.md

@@ -0,0 +1,4 @@
+- The MDX-to-Markdown sync conversion now uses the same fenced-code-block
+  rule as the other sync scripts (hivecommons/docs#569): a backtick run whose
+  info string contains a backtick is no longer mistaken for a fence opener, so
+  it can no longer hide a component's closing tag.

scripts/markdown-fences.ts

@@ -14,16 +14,41 @@ export const DEFAULT_FENCE_LANGUAGE = "text";
 
 const FENCE_RE = /^(\s*)(`{3,}|~{3,})(.*)$/;
 
-interface OpenFence {
-  marker: string;
-}
+/** Where a line sits relative to fenced code blocks. */
+export type FenceLineKind = "outside" | "opener" | "inside" | "closer";
+
+/**
+ * Line-by-line fenced-code-block tracker: the one fence rule every sync script
+ * shares. Feed lines in document order, without their trailing newline. A
+ * fence opens on a run of three or more backticks or tildes — except a
+ * backtick run whose info string contains a backtick, which CommonMark does
+ * not treat as a fence — and closes on a line of the same marker character, at
+ * least as long as the opener, with no info string.
+ */
+export class FenceTracker {
+  private openMarker: string | null = null;
 
-function isFenceCloser(open: OpenFence, marker: string, info: string): boolean {
-  return (
-    marker[0] === open.marker[0] &&
-    marker.length >= open.marker.length &&
-    info.trim() === ""
-  );
+  /** Classify `line` and advance the tracker past it. */
+  next(line: string): FenceLineKind {
+    const m = line.match(FENCE_RE);
+    if (this.openMarker !== null) {
+      if (
+        m &&
+        m[2][0] === this.openMarker[0] &&
+        m[2].length >= this.openMarker.length &&
+        m[3].trim() === ""
+      ) {
+        this.openMarker = null;
+        return "closer";
+      }
+      return "inside";
+    }
+    if (m && !(m[2][0] === "`" && m[3].includes("`"))) {
+      this.openMarker = m[2];
+      return "opener";
+    }
+    return "outside";
+  }
 }
 
 /**
@@ -36,17 +61,10 @@ export function splitFencedSegments(
   content: string
 ): Array<{ text: string; fenced: boolean }> {
   const segments: Array<{ text: string; fenced: boolean }> = [];
-  let open: OpenFence | null = null;
+  const fences = new FenceTracker();
 
   for (const line of content.split(/(?<=\n)/)) {
-    const m = line.replace(/\r?\n$/, "").match(FENCE_RE);
-    let fenced = open !== null;
-    if (open) {
-      if (m && isFenceCloser(open, m[2], m[3])) open = null;
-    } else if (m && !(m[2][0] === "`" && m[3].includes("`"))) {
-      open = { marker: m[2] };
-      fenced = true;
-    }
+    const fenced = fences.next(line.replace(/\r?\n$/, "")) !== "outside";
     const last = segments[segments.length - 1];
     if (last && last.fenced === fenced) last.text += line;
     else segments.push({ text: line, fenced });
@@ -72,14 +90,13 @@ export function mapOutsideFences(
 export function normalizeFences(content: string): string {
   const lines = content.split("\n");
   const out: string[] = [];
-  let open: OpenFence | null = null;
+  const fences = new FenceTracker();
 
   for (const line of lines) {
-    const m = line.match(FENCE_RE);
-    if (open) {
+    const kind = fences.next(line);
+    if (kind === "inside" || kind === "closer") {
       out.push(line);
-      if (m && isFenceCloser(open, m[2], m[3])) {
-        open = null;
+      if (kind === "closer") {
         // MD031: a closer needs a blank line after it. We cannot look ahead
         // without buffering, so the check happens when the next line lands.
         out.push("\0AFTER_FENCE");
@@ -92,21 +109,14 @@ export function normalizeFences(content: string): string {
       if (line.trim() !== "") out.push("");
     }
 
-    if (m) {
-      const [, indent, marker, info] = m;
-      // A backtick fence whose info string contains a backtick is not a
-      // fence per CommonMark; leave it alone.
-      if (marker[0] === "`" && info.includes("`")) {
-        out.push(line);
-        continue;
-      }
+    if (kind === "opener") {
+      const [, indent, marker, info] = line.match(FENCE_RE)!;
       // MD031: blank line before the opener.
       if (out.length > 0 && out[out.length - 1].trim() !== "") out.push("");
       // MD040: language on the opener.
       const lang =
         info.trim() === "" ? DEFAULT_FENCE_LANGUAGE : info.trimStart();
       out.push(`${indent}${marker}${lang}`);
-      open = { marker };
       continue;
     }
 

scripts/mdx-to-markdown.test.ts

@@ -86,6 +86,19 @@ describe("convertMdxToMarkdown", () => {
     );
   });
 
+  it("does not let a backtick run with a backtick in its info string hide a closing tag (CommonMark)", () => {
+    const src = [
+      '<Step number="1" heading="Quote">',
+      "",
+      "  ```not`a`fence",
+      "",
+      "</Step>",
+    ].join("\n");
+    const out = convertMdxToMarkdown(src, { siteBase: SITE });
+    expect(out.markdown).toBe("### Step 1: Quote\n\n```not`a`fence\n");
+    expect(out.notes).toEqual([]);
+  });
+
   it("converts nested <AgentBlock> inside <Step> to labelled sections", () => {
     const src = [
       '<Step number="4" heading="Init">',

scripts/mdx-to-markdown.ts

@@ -22,6 +22,8 @@
  * Fenced code blocks are never parsed for components or rewritten.
  */
 
+import { FenceTracker } from "./markdown-fences";
+
 export interface MdxConvertOptions {
   /** Canonical site the page is published on, e.g. https://spektacular.dev. */
   siteBase: string;
@@ -46,7 +48,6 @@ export interface MdxConversion {
 }
 
 const DEFAULT_IMAGE_PREFIX = "/images/";
-const FENCE_RE = /^\s*(`{3,}|~{3,})(.*)$/;
 // A JSX tag on a line of its own: `<Name attrs>`, `<Name attrs />` or `</Name>`.
 const BLOCK_OPEN_RE = /^\s*<([A-Z][A-Za-z0-9.]*)((?:\s+[^>]*?)?)\s*(\/?)>\s*$/;
 const BLOCK_CLOSE_RE = /^\s*<\/([A-Z][A-Za-z0-9.]*)\s*>\s*$/;
@@ -173,23 +174,9 @@ function dedent(lines: string[]): string[] {
 
 function findClose(lines: string[], from: number, name: string): number {
   let depth = 0;
-  let fence: string | null = null;
+  const fences = new FenceTracker();
   for (let i = from; i < lines.length; i++) {
-    const f = lines[i].match(FENCE_RE);
-    if (fence) {
-      if (
-        f &&
-        f[1][0] === fence[0] &&
-        f[1].length >= fence.length &&
-        f[2].trim() === ""
-      )
-        fence = null;
-      continue;
-    }
-    if (f) {
-      fence = f[1];
-      continue;
-    }
+    if (fences.next(lines[i]) !== "outside") continue;
     const open = lines[i].match(BLOCK_OPEN_RE);
     if (open && open[1] === name && open[3] !== "/") depth++;
     const close = lines[i].match(BLOCK_CLOSE_RE);
@@ -203,24 +190,11 @@ function findClose(lines: string[], from: number, name: string): number {
 
 function convertBlocks(lines: string[], notes: string[]): string[] {
   const out: string[] = [];
-  let fence: string | null = null;
+  const fences = new FenceTracker();
 
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i];
-    const f = line.match(FENCE_RE);
-    if (fence) {
-      out.push(line);
-      if (
-        f &&
-        f[1][0] === fence[0] &&
-        f[1].length >= fence.length &&
-        f[2].trim() === ""
-      )
-        fence = null;
-      continue;
-    }
-    if (f) {
-      fence = f[1];
+    if (fences.next(line) !== "outside") {
       out.push(line);
       continue;
     }