Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
// currency inspection is more careful than theirs so `$6$` / `$5.99$`
// numeric inline math is preserved.

import { mathLikeInlineBody } from "#/lib/math-like-text";

// Inline bracket math: \(x\) or \\(x\\) (JSON-double-escaped from tool outputs).
// Single-line only — remark-math treats inline math the same way.
const BRACKET_INLINE = /\\{1,2}\(([^\n]+?)\\{1,2}\)/g;
Expand Down Expand Up @@ -58,44 +60,6 @@ function restoreCodeRuns(text: string, preserved: string[]): string {
return text.replace(RESTORE, (_, i: string) => preserved[Number(i)] ?? "");
}

/**
* True when the body between two adjacent `$` marks looks like a real math
* expression the model intended. Used to decide whether a currency-signature
* `$` should be escaped or left as a math opener.
*
* | Body | Verdict | Reason |
* | ----------------------------------- | ------- | ----------------------------- |
* | `6`, `5.99`, `1,299` | math | bare numeric literal |
* | `x^2 + 5x`, `n = 3` | math | algebra with short variables |
* | `\frac{a}{b}`, `\sum_{i=1}^n` | math | LaTeX macro present |
* | `5 total`, `today, or `, `5 and 6` | prose | 3+ letter English word |
* | `45 + `, `= 60`, `1,299–` | prose | dangling operator (cut off) |
* | `**bold**`, blank lines, >200 chars | prose | markdown structural / too long |
*/
function mathLikeInlineBody(body: string): boolean {
const t = body.trim();
if (!t) return false;
if (t.length > 200) return false;
// Structural markdown characters bleeding through mean the `$…$` pair
// wrapped around formatted prose, not math.
if (/\*\*|__|~~|\n\n/.test(t)) return false;
// LaTeX command — definitely math.
if (/\\[a-zA-Z]/.test(t)) return true;
// Dangling operator at start or end signals cut-off currency arithmetic
// ("45 + " came from `$45 + $3.60`, not a complete math expression).
if (/[+\-−–×÷=/*→←]\s*$/.test(t)) return false;
if (/^\s*[+\-−–×÷=/*→←]/.test(t)) return false;
// Currency codes should not count as prose words for this test.
const withoutCcy = t.replace(/\b(?:USD|EUR|GBP|JPY|CAD|AUD|CHF|CNY|INR|k|M|B)\b/gi, "");
// A word of 3+ letters is prose ("today", "and", "total") — not math.
if (/[a-zA-Z]{3,}/.test(withoutCcy)) return false;
// Pure numeric with math operators.
if (/^[\s\d.,+\-*/=×÷^_(){}[\]<>|:;'"\\]+$/.test(t)) return true;
// Numeric with 1-2 letter algebraic variables.
if (/^[\sa-zA-Z\d.,+\-*/=×÷^_(){}[\]<>|:;'"\\]+$/.test(t)) return true;
return false;
}

// Rewrite the alternative delimiter dialects some models emit into the
// `$…$` / `$$…$$` form remark-math parses.
export function rewriteAlternativeDelimiters(text: string): string {
Expand Down
40 changes: 40 additions & 0 deletions src/features/workspaces/documents/document-ai-html.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,46 @@ describe("document AI HTML", () => {
expect(html).not.toContain("<sub>");
});

it("rescues dollar-delimited math the model writes as prose", async () => {
// Math renders only as a math node, never as `$…$`. The model reaches for
// dollars by habit, so repair them instead of storing literal text.
const html = await serializeTiptapDocumentToAiHtml(
ensureTiptapDocumentBlockIds(
parseDocumentAiHtml(
"<p>The energy is $E = mc^2$ where $c$ is the speed of light.</p><p>$$\\int_0^1 x^2 dx$$</p>",
),
).document,
);

expect(html).toContain('data-type="inline-math"');
expect(html).toContain('data-latex="E = mc^2"');
expect(html).toContain('data-latex="c"');
expect(html).toContain('data-type="block-math"');
expect(html).toContain('data-latex="\\int_0^1 x^2 dx"');
expect(html).not.toContain("$");
});

it("leaves plain money and prose dollar signs alone", async () => {
const html = await serializeTiptapDocumentToAiHtml(
ensureTiptapDocumentBlockIds(parseDocumentAiHtml("<p>Lunch was $5 and dinner was $10.</p>"))
.document,
);

expect(html).not.toContain("inline-math");
expect(html).toContain("$5 and dinner was $10");
});

it("keeps dollar math inside code and widgets literal", () => {
const document = parseDocumentAiHtml(
'<p>Write <code>$x = 1$</code> in LaTeX.</p><div data-type="widget" title="Demo">const label = "$x$";</div>',
);

const json = JSON.stringify(document);
expect(json).not.toContain("inlineMath");
expect(json).toContain("$x = 1$");
expect(json).toContain("$x$");
});

it("ignores refs supplied in model-authored HTML", async () => {
const html = await serializeTiptapDocumentToAiHtml(
ensureTiptapDocumentBlockIds(parseDocumentAiHtml('<p data-ref="b_modelchosen1">Hello</p>'))
Expand Down
162 changes: 158 additions & 4 deletions src/features/workspaces/documents/document-ai-html.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
tiptapDocumentAiRefAttribute,
} from "#/features/workspaces/documents/tiptap-schema";
import { sha256Base64UrlText } from "#/lib/binary";
import { mathLikeInlineBody } from "#/lib/math-like-text";

const TEXT_NODE = 3;
const documentBlockIdPattern = /^b_[A-Za-z0-9_-]{12}$/;
Expand Down Expand Up @@ -210,19 +211,21 @@ function serializeTiptapFragmentToAiHtml(fragment: Fragment) {
}

/**
* Rescues the two tags ProseMirror parses lossily.
* Rescues the tags and math dialects ProseMirror parses lossily.
*
* Everything else outside the schema is already handled by the parser: unknown
* wrappers are skipped with their children kept, and `<script>`/`<style>` text
* never reaches the document. These two lose meaning instead:
* never reaches the document. These lose meaning instead:
* - `<h5>`/`<h6>` fall all the way to a paragraph, so the heading disappears;
* the schema stops at 4, so 4 is where they belong.
* - `<sub>`/`<sup>` flatten to bare text (`CH<sub>4</sub>` becomes `CH4`), so
* they become inline math instead. `{}_{4}` is the KaTeX form for a
* subscript with no base, which is what the tag means on its own.
* - `$…$`/`$$…$$` in prose stay literal text, because math renders only as a
* math node, never delimiters. They become inline and block math instead.
*
* Models write both by habit — evals showed `CH<sub>4</sub>` surviving an
* explicit instruction not to use it.
* Models write all of these by habit — evals showed `CH<sub>4</sub>` and dollar
* math surviving an explicit instruction not to use them.
*/
function rewriteLossyElements(htmlDocument: Document) {
for (const element of htmlDocument.body.querySelectorAll("h5, h6")) {
Expand All @@ -248,6 +251,157 @@ function rewriteLossyElements(htmlDocument: Document) {
);
element.replaceWith(math);
}

rewriteDollarMath(htmlDocument);
}

interface DollarMathSegment {
type: "text" | "inline" | "block";
value: string;
}

/**
* Converts `$…$` and `$$…$$` prose into math nodes, the form the document
* schema renders. Chat repairs the same dialect (`normalize-llm-markdown.ts`);
* documents never got the equivalent, so the model's inline math stayed literal.
*
* Code, widget source, and math nodes keep their raw text — a LaTeX example in a
* code block must not turn into an equation. Money stays plain: an inline pair
* only converts when its body reads as math (`mathLikeInlineBody`), so `$5 and
* $10` is left alone. A `$$…$$` pair is display math with no money reading, so a
* non-empty body is enough.
*/
function rewriteDollarMath(htmlDocument: Document) {
const textNodes: globalThis.Node[] = [];
collectDollarMathTextNodes(htmlDocument.body, textNodes);

for (const textNode of textNodes) {
const segments = splitDollarMath(textNode.textContent ?? "");
if (!segments) {
continue;
}

const parent = textNode.parentNode;
if (!parent) {
continue;
}

for (const segment of segments) {
parent.insertBefore(createDollarMathNode(htmlDocument, segment), textNode);
}
parent.removeChild(textNode);
}
}

function collectDollarMathTextNodes(root: globalThis.Node, out: globalThis.Node[]) {
for (const child of Array.from(root.childNodes)) {
if (child.nodeType === TEXT_NODE) {
if (child.textContent?.includes("$")) {
out.push(child);
}
continue;
}
if (isDollarMathSkippedElement(child)) {
continue;
}
collectDollarMathTextNodes(child, out);
}
}

function isDollarMathSkippedElement(node: globalThis.Node) {
const element = node as Element;
if (typeof element.getAttribute !== "function") {
return false;
}
const tag = element.tagName?.toLowerCase();
if (tag === "code" || tag === "pre") {
return true;
}
const dataType = element.getAttribute("data-type");
return dataType === "widget" || dataType === "inline-math" || dataType === "block-math";
}

function createDollarMathNode(htmlDocument: Document, segment: DollarMathSegment): globalThis.Node {
if (segment.type === "text") {
return htmlDocument.createTextNode(segment.value) as unknown as globalThis.Node;
}
const element = htmlDocument.createElement(segment.type === "block" ? "div" : "span");
element.setAttribute("data-type", segment.type === "block" ? "block-math" : "inline-math");
element.setAttribute("data-latex", segment.value);
return element as unknown as globalThis.Node;
}

/**
* Splits a text run into plain-text and math segments. Returns null when nothing
* converts, so the caller leaves the original text node untouched.
*/
function splitDollarMath(text: string): DollarMathSegment[] | null {
const segments: DollarMathSegment[] = [];
let buffer = "";
let index = 0;

const flushBuffer = () => {
if (buffer) {
segments.push({ type: "text", value: buffer });
buffer = "";
}
};

while (index < text.length) {
if (text[index] !== "$") {
buffer += text[index];
index += 1;
continue;
}

if (text[index + 1] === "$") {
const close = text.indexOf("$$", index + 2);
const body = close === -1 ? "" : text.slice(index + 2, close).trim();
if (body) {
flushBuffer();
segments.push({ type: "block", value: body });
index = close + 2;
continue;
}
buffer += "$$";
index += 2;
continue;
}

const close = findInlineMathClose(text, index + 1);
if (close !== -1) {
const body = text.slice(index + 1, close);
if (mathLikeInlineBody(body)) {
flushBuffer();
segments.push({ type: "inline", value: body.trim() });
index = close + 1;
continue;
}
}
buffer += "$";
index += 1;
}

// Only math segments break the flow; if none were produced the buffer holds
// the whole text, so leave the original text node untouched.
if (segments.length === 0) {
return null;
}
flushBuffer();
return segments;
}

/** Index of the closing `$` for an inline pair, or -1. Stops at a line break. */
function findInlineMathClose(text: string, start: number) {
for (let index = start; index < text.length; index += 1) {
if (text[index] === "\n") {
return -1;
}
if (text[index] === "$") {
return index === start ? -1 : index;
}
}
return -1;
}

function validateDocumentAiHtml(root: HTMLElement) {
Expand Down
38 changes: 38 additions & 0 deletions src/lib/math-like-text.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/**
* True when the body between two `$` marks reads as a real math expression the
* model intended, rather than money or prose. Both the chat Markdown normalizer
* and the document HTML parser use it to tell an inline math pair apart from a
* `$5 and $10` currency pair.
*
* | Body | Verdict | Reason |
* | ----------------------------------- | ------- | ----------------------------- |
* | `6`, `5.99`, `1,299` | math | bare numeric literal |
* | `x^2 + 5x`, `n = 3` | math | algebra with short variables |
* | `\frac{a}{b}`, `\sum_{i=1}^n` | math | LaTeX macro present |
* | `5 total`, `today, or `, `5 and 6` | prose | 3+ letter English word |
* | `45 + `, `= 60`, `1,299–` | prose | dangling operator (cut off) |
* | `**bold**`, blank lines, >200 chars | prose | markdown structural / too long |
*/
export function mathLikeInlineBody(body: string): boolean {
const t = body.trim();
if (!t) return false;
if (t.length > 200) return false;
// Structural markdown characters bleeding through mean the `$…$` pair
// wrapped around formatted prose, not math.
if (/\*\*|__|~~|\n\n/.test(t)) return false;
// LaTeX command — definitely math.
if (/\\[a-zA-Z]/.test(t)) return true;
// Dangling operator at start or end signals cut-off currency arithmetic
// ("45 + " came from `$45 + $3.60`, not a complete math expression).
if (/[+\-−–×÷=/*→←]\s*$/.test(t)) return false;
if (/^\s*[+\-−–×÷=/*→←]/.test(t)) return false;
// Currency codes should not count as prose words for this test.
const withoutCcy = t.replace(/\b(?:USD|EUR|GBP|JPY|CAD|AUD|CHF|CNY|INR|k|M|B)\b/gi, "");
// A word of 3+ letters is prose ("today", "and", "total") — not math.
if (/[a-zA-Z]{3,}/.test(withoutCcy)) return false;
// Pure numeric with math operators.
if (/^[\s\d.,+\-*/=×÷^_(){}[\]<>|:;'"\\]+$/.test(t)) return true;
// Numeric with 1-2 letter algebraic variables.
if (/^[\sa-zA-Z\d.,+\-*/=×÷^_(){}[\]<>|:;'"\\]+$/.test(t)) return true;
return false;
}