diff --git a/AGENTS.md b/AGENTS.md index b3a2f11..efd5d6e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -142,10 +142,6 @@ Nesting the markdown exposes a few pipeline limits: - A `types.ts` outside an indexed docs section must pass `{ excludeFromIndex: true }`, or the types loader creates a parent index page (`docs/app/page.mdx` shadowed the landing route). It must also import the component by relative path; the loader resolves `@/` as a package. -- In fenced code, a trailing `// @highlight` on the **last** line drops that line, and - `createDemo(import.meta.url, X, {` followed by emphasis comments disables them for the - block. Wrap the statement in `@highlight-start` / `@highlight-end`, or put each argument - on its own line. - The landing demos use their own `createDemo` (`docs/app/(shared)/demos/createDemo.ts`) with `fallbackUsesExtraFiles`, so the page backs its claim that every file of a demo is in the initial HTML as plain text. A `ContentLoading` must take `extraSource` from diff --git a/packages/docs/src/pipeline/enhanceCodeEmphasis/enhanceCodeEmphasis.test.ts b/packages/docs/src/pipeline/enhanceCodeEmphasis/enhanceCodeEmphasis.test.ts index 8845d32..6f53ac3 100644 --- a/packages/docs/src/pipeline/enhanceCodeEmphasis/enhanceCodeEmphasis.test.ts +++ b/packages/docs/src/pipeline/enhanceCodeEmphasis/enhanceCodeEmphasis.test.ts @@ -119,7 +119,7 @@ const e = 5; // @highlight`, const b = 2; const c = 3; const d = 4; - " + const e = 5;" `, ); }); @@ -488,7 +488,7 @@ const e = 5; // @highlight`, ); expect(result).toMatchInlineSnapshot( - `"<Input value={a} /> <Input value={b} /> // @highlight-text "value""`, + `"<Input value={a} /> <Input value={b} />"`, ); }); @@ -530,7 +530,8 @@ const e = 5; // @highlight`, createEnhanceCodeEmphasis({ strictHighlightText: true }), ); - expect(result).toContain(''); + // The element itself becomes the mark, keeping its classes. + expect(result).toContain('value'); expect(result).not.toContain('data-hl-part'); }); @@ -623,7 +624,7 @@ const another = 99; // @highlight`, const y = 2; return x + y; } - " + const another = 99;" `); }); diff --git a/packages/docs/src/pipeline/loaderUtils/optimization.test.ts b/packages/docs/src/pipeline/loaderUtils/optimization.test.ts index e3e43e8..020d3d4 100644 --- a/packages/docs/src/pipeline/loaderUtils/optimization.test.ts +++ b/packages/docs/src/pipeline/loaderUtils/optimization.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, vi } from 'vitest'; import { resolveModulePath } from './resolveModulePath'; import type { DirectoryEntry } from './resolveModulePath'; +import { parseImportsAndComments } from './parseImportsAndComments'; describe('Filesystem Optimization Tests', () => { it('should make only one directory read when includeTypeDefs is true', async () => { @@ -129,3 +130,102 @@ describe('Filesystem Optimization Tests', () => { }); }); }); + +describe('parseImportsAndComments scales linearly', () => { + // Each input once made the parser rescan earlier text for every position or + // comment, so its parse time grew with the square of its size. In one pass, + // parsing an input 16 times as large takes about 16 times as long; a rescan + // takes about 256 times as long. The limit sits between the two, so the guard + // doesn't depend on how fast the machine is, and taking the fastest of several + // runs at each size keeps one-off pauses (JIT compilation, GC) out of it. + const growth = 16; + const maxTimeGrowth = 64; + const options = { + removeCommentsWithPrefix: ['@highlight'], + notableCommentsPrefix: ['@highlight'], + }; + + /** + * Parses `build(count)` and `build(count * growth)` several times, returning how + * many times as long the larger input took, and its result. + */ + function measureTimeGrowth(build: (count: number) => string, count: number, fileName: string) { + const inputs = [build(count), build(count * growth)]; + const fastestMs = [Infinity, Infinity]; + // The larger input is parsed last in each round, so this ends as its result + let result: ReturnType | undefined; + for (let round = 0; round < 7; round += 1) { + for (let index = 0; index < inputs.length; index += 1) { + const start = performance.now(); + result = parseImportsAndComments(inputs[index], fileName, options); + fastestMs[index] = Math.min(fastestMs[index], performance.now() - start); + } + } + return { timeGrowth: fastestMs[1] / fastestMs[0], result: result! }; + } + + it('reads an MDX import after a very long line', () => { + const { timeGrowth, result } = measureTimeGrowth( + (count) => `${' '.repeat(count)}x\nimport { Button } from './Button';\n`, + 5_000, + '/src/demo.mdx', + ); + + expect(Object.keys(result.relative)).toEqual(['./Button']); + expect(timeGrowth).toBeLessThan(maxTimeGrowth); + }); + + it('strips and keeps many comments on one long line', () => { + const { timeGrowth, result } = measureTimeGrowth( + (count) => 'const count = 1; /* @highlight */ /* note */ '.repeat(count), + 300, + '/src/demo.js', + ); + + expect(result.code).toBe('const count = 1; /* note */ '.repeat(300 * growth)); + expect(timeGrowth).toBeLessThan(maxTimeGrowth); + }); + + it('maps the positions of many imports in stripped code', () => { + const count = 250; + const { timeGrowth, result } = measureTimeGrowth( + (importCount) => + Array.from( + { length: importCount }, + (_, index) => `import { Item${index} } from './item${index}'; // @highlight`, + ).join('\n'), + count, + '/src/demo.ts', + ); + + const lastIndex = count * growth - 1; + const lastPath = `'./item${lastIndex}'`; + const lastStart = result.code!.indexOf(lastPath); + expect(result.relative[`./item${lastIndex}`].positions).toEqual([ + { start: lastStart, end: lastStart + lastPath.length }, + ]); + expect(timeGrowth).toBeLessThan(maxTimeGrowth); + }); + + it('ends unfinished MDX ESM blocks without searching the rest of the file for each', () => { + const { timeGrowth, result } = measureTimeGrowth( + (count) => "export const loaders = {\n load: () => import('./chart'),\n\n".repeat(count), + 250, + '/src/demo.mdx', + ); + + expect(Object.keys(result.relative)).toEqual(['./chart']); + expect(timeGrowth).toBeLessThan(maxTimeGrowth); + }); + + it('gives up on an unclosed export list without searching the rest of the file', () => { + const { timeGrowth, result } = measureTimeGrowth( + (count) => 'export { first, second\nconst count = 1;\n'.repeat(count), + 250, + '/src/demo.ts', + ); + + expect(result.relative).toEqual({}); + expect(timeGrowth).toBeLessThan(maxTimeGrowth); + }); +}); diff --git a/packages/docs/src/pipeline/loaderUtils/parseImportsAndComments.test.ts b/packages/docs/src/pipeline/loaderUtils/parseImportsAndComments.test.ts index 71d078c..f6219bd 100644 --- a/packages/docs/src/pipeline/loaderUtils/parseImportsAndComments.test.ts +++ b/packages/docs/src/pipeline/loaderUtils/parseImportsAndComments.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { parseImportsAndComments } from './parseImportsAndComments'; +import type { ImportName } from './parseImportsAndComments'; describe('parseImportsAndComments', () => { it('should resolve relative import paths and group by import path', async () => { @@ -332,6 +333,322 @@ describe('parseImportsAndComments', () => { }); }); + describe('imports without semicolons', () => { + /** Position of a quoted module path in `code`, as the parser reports it. */ + function positionOf(code: string, quotedPath: string, fromIndex = 0) { + const start = code.indexOf(quotedPath, fromIndex); + return { start, end: start + quotedPath.length }; + } + + it('detects every import form when each statement ends at a line break', () => { + const code = `import * as React from 'react' +import Button, { type ButtonProps } from './Button' +import type { Theme } from './theme' +import { + Checkbox, + Radio as RadioInput, +} from './inputs' +import './setup' +import styles from './Demo.module.css' + +export function Demo() { + return + // @focus-end +}`; + + const result = parseImportsAndComments(code, '/src/Counter.tsx', { + removeCommentsWithPrefix: ['@highlight', '@focus'], + notableCommentsPrefix: ['@highlight', '@focus'], + }); + + // The same code and line keys as the semicolon-terminated source produces. + expect(result.code).toBe(`import * as React from 'react' +import styles from './Counter.module.css' + +export function Counter() { + const [count, setCount] = React.useState(0) + return +}`); + expect(result.comments).toEqual({ + 2: ['@highlight'], + 5: ['@focus-start', '@highlight'], + 7: ['@focus-end'], + }); + expect(Object.keys(result.relative)).toEqual(['./Counter.module.css']); + }); }); // Test cases for export-from statements @@ -3420,6 +4295,56 @@ describe('Export-from statement parsing', () => { }); }); + it('ignores a `from` in the body of an exported declaration', () => { + const withoutSemicolons = `import * as React from 'react' + +export function Button() { + // Adapted from "a design guide" + const label = 'from the theme' + return

Loaded from './data.json'

+} + +export { Checkbox } from './Checkbox' +`; + const withSemicolons = `import * as React from 'react'; + +export function Button() { + // Adapted from "a design guide" + const label = 'from the theme'; + return

Loaded from './data.json'

; +} + +export { Checkbox } from './Checkbox'; +`; + + for (const code of [withoutSemicolons, withSemicolons]) { + const result = parseImportsAndComments(code, '/src/Button.tsx'); + expect(Object.keys(result.relative)).toEqual(['./Checkbox']); + expect(Object.keys(result.externals)).toEqual(['react']); + } + }); + + it('should parse namespace and type-only star re-exports', () => { + const code = `export * as Icons from './icons' +export type * from './types' +`; + const result = parseImportsAndComments(code, '/src/index.ts'); + + expect(result.relative).toEqual({ + './icons': { + url: 'file:///src/icons', + names: [{ name: 'Icons', type: 'namespace' }], + positions: [{ start: 23, end: 32 }], + }, + './types': { + url: 'file:///src/types', + names: [], + includeTypeDefs: true, + positions: [{ start: 52, end: 61 }], + }, + }); + }); + it('should parse mixed import and export-from statements', async () => { const code = ` import React from 'react'; @@ -3507,6 +4432,11 @@ describe('Export-from statement parsing', () => { names: [{ name: 'Component', type: 'named' }], positions: [{ start: 26, end: 39 }], }, + './Button': { + url: 'file:///src/Button', + names: [{ name: 'Button', type: 'named' }], + positions: [{ start: 63, end: 73 }], + }, }, externals: {}, }); diff --git a/packages/docs/src/pipeline/loaderUtils/parseImportsAndComments.ts b/packages/docs/src/pipeline/loaderUtils/parseImportsAndComments.ts index 717e24c..b921b0b 100644 --- a/packages/docs/src/pipeline/loaderUtils/parseImportsAndComments.ts +++ b/packages/docs/src/pipeline/loaderUtils/parseImportsAndComments.ts @@ -156,19 +156,224 @@ function stripCommentMarkers(commentText: string): string[] { } /** - * Counts consecutive backticks starting at a given position (used for MDX code blocks). - * @param sourceText - The source text to scan - * @param startPos - The position to start counting from - * @returns The number of consecutive backticks found + * Counts how many times `char` repeats starting at `start`. */ -function countBackticks(sourceText: string, startPos: number): number { - let count = 0; - let pos = startPos; - while (pos < sourceText.length && sourceText[pos] === '`') { - count += 1; +function countRepeated(text: string, start: number, char: string): number { + let pos = start; + while (text[pos] === char) { pos += 1; } - return count; + return pos - start; +} + +/** + * Whether nothing but whitespace follows `pos` up to the end of its line. + */ +function isRestOfLineBlank(text: string, pos: number): boolean { + let cursor = pos; + while (cursor < text.length && text[cursor] !== '\n') { + if (!/\s/.test(text[cursor])) { + return false; + } + cursor += 1; + } + return true; +} + +/** + * Returns the length of the MDX code fence that opens at `pos`, or `0` when none + * does. Following CommonMark, a fence is a run of at least three backticks or + * three tildes that starts its line, and a backtick fence's info string can't + * contain a backtick (that's inline code instead). MDX has no indented code + * blocks, so any indentation may come before it. + */ +function getCodeFenceLength(text: string, pos: number): number { + const marker = text[pos]; + if ((marker !== '`' && marker !== '~') || !isAtLineStart(text, pos)) { + return 0; + } + const length = countRepeated(text, pos, marker); + if (length < 3) { + return 0; + } + if (marker === '`') { + const lineEnd = text.indexOf('\n', pos); + const info = text.slice(pos + length, lineEnd === -1 ? text.length : lineEnd); + if (info.includes('`')) { + return 0; + } + } + return length; +} + +/** + * What a code span search saw of the rest of its paragraph: where it stopped, + * and where the last run of backticks of each length starts before that. + */ +interface CodeSpanSearch { + stop: number; + lastRunStarts: Map; +} + +/** + * Finds the end of an MDX inline code span whose opening run of `length` + * backticks ends at `pos`: just past the next run of exactly `length` backticks. + * A span stays within its paragraph, so a blank line or a line that opens a code + * fence ends the search. Returns `-1` when there's no closing run, in which case + * the opening backticks are plain text. Records what it saw in `search`. + */ +function findCodeSpanEnd( + text: string, + pos: number, + length: number, + search: CodeSpanSearch, +): number { + search.lastRunStarts.clear(); + let cursor = pos; + while (cursor < text.length) { + if (text[cursor] === '\n') { + const lineStart = cursor + 1; + let firstChar = lineStart; + while (text[firstChar] === ' ' || text[firstChar] === '\t') { + firstChar += 1; + } + if (isRestOfLineBlank(text, lineStart) || getCodeFenceLength(text, firstChar) > 0) { + break; + } + cursor = lineStart; + } else if (text[cursor] === '`') { + const runLength = countRepeated(text, cursor, '`'); + if (runLength === length) { + return cursor + runLength; + } + search.lastRunStarts.set(runLength, cursor); + cursor += runLength; + } else { + cursor += 1; + } + } + search.stop = cursor; + return -1; +} + +/** + * Whether only spaces or tabs come before `pos` on its line. + */ +function isAtLineStart(text: string, pos: number): boolean { + let cursor = pos; + while (cursor > 0 && (text[cursor - 1] === ' ' || text[cursor - 1] === '\t')) { + cursor -= 1; + } + return cursor === 0 || text[cursor - 1] === '\n'; +} + +/** + * Whether an MDX ESM block starts at `lineStart`. As in MDX, the line must begin + * with `import` or `export` followed by a space. (Indentation before the keyword + * is also accepted.) + */ +function isMdxEsmStart(text: string, lineStart: number): boolean { + let cursor = lineStart; + while (text[cursor] === ' ' || text[cursor] === '\t') { + cursor += 1; + } + return ( + (text.startsWith('import', cursor) || text.startsWith('export', cursor)) && + text[cursor + 6] === ' ' + ); +} + +/** + * Returns where the MDX ESM block that starts at `lineStart` ends, or `lineStart` + * when the line doesn't start one (see `isMdxEsmStart`). As in MDX, a block ends + * at a blank line where its code is complete: outside any bracket, string, + * template literal or comment. So a blank line between the properties of an + * exported object doesn't end it. When the code never completes (a syntax error, + * which MDX rejects), the block ends at its first blank line instead, and + * `incomplete` is set. With `continueIncomplete` false, the block ends at its first + * blank line regardless. + */ +function findMdxEsmEnd( + text: string, + lineStart: number, + continueIncomplete: boolean, +): { end: number; incomplete: boolean } { + if (!isMdxEsmStart(text, lineStart)) { + return { end: lineStart, incomplete: false }; + } + const len = text.length; + let state: 'code' | 'string' | 'template' | 'line-comment' | 'block-comment' = 'code'; + let quote = ''; + // Open brackets, and the bracket depth at which each open `${…}` of a template + // literal began + let depth = 0; + const templateDepths: number[] = []; + let firstBlankLineEnd = -1; + let cursor = lineStart; + while (cursor < len) { + const ch = text[cursor]; + if (ch === '\n') { + // A string can't hold a line break, nor a line comment run past one. + if (state === 'string' || state === 'line-comment') { + state = 'code'; + } + if (isRestOfLineBlank(text, cursor + 1)) { + const complete = state === 'code' && depth === 0 && templateDepths.length === 0; + if (complete || !continueIncomplete) { + return { end: cursor + 1, incomplete: false }; + } + if (firstBlankLineEnd === -1) { + firstBlankLineEnd = cursor + 1; + } + } + cursor += 1; + } else if (state === 'string' || state === 'template') { + if (ch === '\\') { + cursor += 2; + continue; + } + if (state === 'string' && ch === quote) { + state = 'code'; + } else if (state === 'template' && ch === '`') { + state = 'code'; + } else if (state === 'template' && ch === '$' && text[cursor + 1] === '{') { + templateDepths.push(depth); + state = 'code'; + cursor += 1; + } + cursor += 1; + } else if (state === 'line-comment') { + cursor += 1; + } else if (state === 'block-comment') { + if (ch === '*' && text[cursor + 1] === '/') { + state = 'code'; + cursor += 1; + } + cursor += 1; + } else { + if (ch === '/' && (text[cursor + 1] === '/' || text[cursor + 1] === '*')) { + state = text[cursor + 1] === '/' ? 'line-comment' : 'block-comment'; + cursor += 1; + } else if (ch === "'" || ch === '"') { + state = 'string'; + quote = ch; + } else if (ch === '`') { + state = 'template'; + } else if (ch === '{' || ch === '(' || ch === '[') { + depth += 1; + } else if (ch === '}' && templateDepths[templateDepths.length - 1] === depth) { + templateDepths.pop(); + state = 'template'; + } else if ((ch === '}' || ch === ')' || ch === ']') && depth > 0) { + depth -= 1; + } + cursor += 1; + } + } + if (state !== 'template' && state !== 'block-comment' && depth === 0) { + return { end: len, incomplete: false }; + } + return { end: firstBlankLineEnd === -1 ? len : firstBlankLineEnd, incomplete: true }; } /** @@ -204,13 +409,56 @@ function scanForImports( const shouldProcessComments = !!(removeCommentsWithPrefix || notableCommentsPrefix); // Only map positions when actually stripping comments (code will differ from source) const shouldMapPositions = !!removeCommentsWithPrefix; - let result = shouldProcessComments ? '' : sourceCode; + // The processed output (after comment removal), kept as its finished lines plus + // its current line, which is split into everything up to its last + // non-whitespace character and the whitespace after that. A comment can then + // take back, trim or check its line in constant time, however long the line. + let finishedOutput = ''; + let lineOutput = ''; + let lineTrailingSpace = ''; + const appendOutput = (text: string) => { + const lineStart = text.lastIndexOf('\n') + 1; + if (lineStart > 0) { + finishedOutput += lineOutput + lineTrailingSpace + text.slice(0, lineStart); + lineOutput = ''; + lineTrailingSpace = ''; + } + let contentEnd = text.length; + while (contentEnd > lineStart && isWhitespace(text[contentEnd - 1])) { + contentEnd -= 1; + } + if (contentEnd > lineStart) { + lineOutput += lineTrailingSpace + text.slice(lineStart, contentEnd); + lineTrailingSpace = text.slice(contentEnd); + } else { + lineTrailingSpace += text.slice(lineStart); + } + }; + // `appendOutput` for a single character, the scanner's most common append. + const appendOutputChar = (ch: string) => { + if (ch === '\n') { + finishedOutput += lineOutput + lineTrailingSpace + ch; + lineOutput = ''; + lineTrailingSpace = ''; + } else if (isWhitespace(ch)) { + lineTrailingSpace += ch; + } else { + lineOutput += lineTrailingSpace + ch; + lineTrailingSpace = ''; + } + }; // Track whether any comment was actually stripped (not just that the option was provided) let anyCommentStripped = false; - // Position mapping from original source to processed source (after comment removal) - const positionMapping = new Map(); - let processedPos = 0; + // Where each detected statement starts in the source and in the processed + // output. A statement is copied to the output unchanged, so a position inside + // one maps by its offset from the statement's start. + const statementSourceStarts: number[] = []; + const statementOutputStarts: number[] = []; + let detectionSourceStart = 0; + let detectionOutputStart = 0; + const detectionPositionMapper = (originalPos: number): number => + shouldMapPositions ? detectionOutputStart + (originalPos - detectionSourceStart) : originalPos; // Helper to check if a comment matches notable prefix const matchesNotablePrefix = (commentText: string): boolean => { @@ -240,12 +488,45 @@ function scanForImports( | 'template' | 'codeblock' = 'code'; let stringQuote: string | null = null; - let codeblockBacktickCount = 0; // Track how many backticks opened the current code block + // The marker and length of the fence that opened the current MDX code block + let codeFenceMarker = ''; + let codeFenceLength = 0; + // What the last failed code span search saw of its paragraph. A later opening + // run there that's the last run of its length has no closing run either, so it's + // known to be plain text without searching again: each paragraph is searched + // through at most once for unclosed runs. + let failedCodeSpanSearch: CodeSpanSearch = { stop: -1, lastRunStarts: new Map() }; + let codeSpanSearch: CodeSpanSearch = { stop: -1, lastRunStarts: new Map() }; + // Where the current MDX ESM block ends. The block is JavaScript; the rest of + // an MDX document outside code blocks is prose. + let mdxEsmEnd = 0; + // Whether an ESM block can run on past a blank line inside unfinished code. Off + // once a block's code never finishes (invalid MDX), so the blocks after it end + // at their first blank line and the text is searched to its end only once. + let mdxEsmContinues = true; // Comment stripping variables let commentStart = 0; let commentStartOutputLine = 0; - let lineStartPos = 0; + // The output line taken back when a comment starts: its content up to the last + // non-whitespace character, and the whitespace after that let preCommentContent = ''; + let preCommentTrailingSpace = ''; + + // Takes back what the current output line already holds, so a comment's + // handler can re-add it (trimmed, or not at all for a comment-only line). It's + // read from the output rather than the source: a comment stripped earlier on + // the same line is already gone from the output. + const takeBackOutputLine = () => { + preCommentContent = lineOutput; + preCommentTrailingSpace = lineTrailingSpace; + lineOutput = ''; + lineTrailingSpace = ''; + }; + // Puts the taken-back line back, with or without its trailing whitespace. + const restoreOutputLine = (keepTrailingSpace: boolean) => { + lineOutput = preCommentContent; + lineTrailingSpace = keepTrailingSpace ? preCommentTrailingSpace : ''; + }; while (i < len) { const ch = sourceCode[i]; @@ -255,27 +536,52 @@ function scanForImports( // Track line numbers for newlines in code if (ch === '\n') { if (shouldProcessComments) { - result += ch; - processedPos += 1; + appendOutputChar(ch); } outputLine += 1; - lineStartPos = i + 1; i += 1; continue; } - // Check for backtick sequences (3 or more backticks start code blocks in MDX) - if (isMdxFile && ch === '`') { - // Count consecutive backticks - const backtickCount = countBackticks(sourceCode, i); - if (backtickCount >= 3) { + if (isMdxFile && i >= mdxEsmEnd && (i === 0 || sourceCode[i - 1] === '\n')) { + const esmBlock = findMdxEsmEnd(sourceCode, i, mdxEsmContinues); + mdxEsmEnd = esmBlock.end; + if (esmBlock.incomplete) { + mdxEsmContinues = false; + } + } + const inMdxProse = isMdxFile && i >= mdxEsmEnd; + + // In MDX prose, a code fence opens a code block; any other run of backticks + // is inline code, which ends at the next run of the same length. + if (inMdxProse && (ch === '`' || ch === '~')) { + const fenceLength = getCodeFenceLength(sourceCode, i); + const backtickCount = ch === '`' ? countRepeated(sourceCode, i, '`') : 0; + let skipTo = i; + if (fenceLength > 0) { state = 'codeblock'; - codeblockBacktickCount = backtickCount; + codeFenceMarker = ch; + codeFenceLength = fenceLength; + skipTo = i + fenceLength; + } else if (backtickCount > 0) { + const knownUnclosed = + i < failedCodeSpanSearch.stop && + failedCodeSpanSearch.lastRunStarts.get(backtickCount) === i; + const spanEnd = knownUnclosed + ? -1 + : findCodeSpanEnd(sourceCode, i + backtickCount, backtickCount, codeSpanSearch); + if (spanEnd === -1 && !knownUnclosed) { + [failedCodeSpanSearch, codeSpanSearch] = [codeSpanSearch, failedCodeSpanSearch]; + } + skipTo = spanEnd === -1 ? i + backtickCount : spanEnd; + } + if (skipTo > i) { + const skipped = sourceCode.slice(i, skipTo); if (shouldProcessComments) { - result += sourceCode.slice(i, i + backtickCount); - processedPos += backtickCount; + appendOutput(skipped); } - i += backtickCount; + outputLine += skipped.split('\n').length - 1; + i = skipTo; continue; } } @@ -284,11 +590,7 @@ function scanForImports( if (shouldProcessComments) { commentStart = i; commentStartOutputLine = outputLine; - // Remove content that was already added to result for this line - const contentSinceLineStart = sourceCode.slice(lineStartPos, commentStart); - result = result.slice(0, result.length - contentSinceLineStart.length); - processedPos -= contentSinceLineStart.length; - preCommentContent = contentSinceLineStart; + takeBackOutputLine(); } state = 'singleline-comment'; i += 2; @@ -299,65 +601,54 @@ function scanForImports( if (shouldProcessComments) { commentStart = i; commentStartOutputLine = outputLine; - // Remove content that was already added to result for this line - const contentSinceLineStart = sourceCode.slice(lineStartPos, commentStart); - result = result.slice(0, result.length - contentSinceLineStart.length); - processedPos -= contentSinceLineStart.length; - preCommentContent = contentSinceLineStart; + takeBackOutputLine(); } state = 'multiline-comment'; i += 2; continue; } // Start of string - if (isStringStart(ch, isMdxFile)) { + if (isStringStart(ch, inMdxProse)) { state = ch === '`' ? 'template' : 'string'; stringQuote = ch; if (shouldProcessComments) { - result += ch; - processedPos += 1; + appendOutputChar(ch); } i += 1; continue; } - // Update position mapping for current position - if (shouldProcessComments) { - positionMapping.set(i, processedPos); + // Record where a statement found here would start, for mapping positions + // inside it to the processed output. + if (shouldMapPositions) { + detectionSourceStart = i; + detectionOutputStart = finishedOutput.length + lineOutput.length + lineTrailingSpace.length; } - // Create position mapper function - const positionMapper = (originalPos: number): number => { - if (!shouldMapPositions) { - return originalPos; // No comment stripping, positions are unchanged - } - // Find the closest mapped position - let closest = 0; - positionMapping.forEach((procPos, origPos) => { - if (origPos <= originalPos && origPos > closest) { - closest = origPos; - } - }); - const offset = originalPos - closest; - return (positionMapping.get(closest) || 0) + offset; - }; - - // Use the provided import detector on the original source code - const detection = importDetector(sourceCode, i, positionMapper); + // Use the provided import detector on the original source code. MDX prose + // holds no statements: any `import` or `export` there is text (e.g. "each + // import's path", or `import()` in a sentence), which must not swallow the + // text after it — and with it the start of a code block. A line that starts + // ESM begins an ESM block, where every import counts. + const detection = inMdxProse + ? { found: false, nextPos: i } + : importDetector(sourceCode, i, detectionPositionMapper); if (detection.found) { if (detection.statement) { statements.push(detection.statement); } + if (shouldMapPositions) { + statementSourceStarts.push(detectionSourceStart); + statementOutputStarts.push(detectionOutputStart); + } // Copy the detected import to result if we're building one if (shouldProcessComments) { const importText = sourceCode.slice(i, detection.nextPos); - result += importText; - processedPos += importText.length; + appendOutput(importText); // Count newlines in multi-line imports to keep outputLine accurate for (let j = 0; j < importText.length; j += 1) { if (importText[j] === '\n') { outputLine += 1; - lineStartPos = i + j + 1; } } } @@ -366,8 +657,7 @@ function scanForImports( } if (shouldProcessComments) { - result += ch; - processedPos += 1; + appendOutputChar(ch); } i += 1; continue; @@ -399,28 +689,27 @@ function scanForImports( if (shouldStrip) { anyCommentStripped = true; // Check if comment is the only thing on its line (ignoring whitespace) - const isCommentOnlyLine = preCommentContent.trim() === ''; + const isCommentOnlyLine = preCommentContent === ''; if (isCommentOnlyLine) { // Don't add the pre-comment content or newline for comment-only lines // Skip the newline entirely } else { - // Comment is inline, keep the pre-comment content (with trailing whitespace trimmed) and newline - result += preCommentContent.trimEnd(); - result += '\n'; - processedPos += preCommentContent.trimEnd().length + 1; + // Comment is inline, keep the pre-comment content (with trailing whitespace trimmed) and newline. + // A CRLF line's `\r` is the comment's last character, so restore it with the newline. + const lineBreak = sourceCode[i - 1] === '\r' ? '\r\n' : '\n'; + restoreOutputLine(false); + appendOutput(lineBreak); outputLine += 1; } } else { // Keep the comment and newline - result += preCommentContent; - result += commentText; - result += '\n'; - processedPos += preCommentContent.length + commentText.length + 1; + restoreOutputLine(true); + appendOutput(commentText); + appendOutput('\n'); outputLine += 1; } preCommentContent = ''; - lineStartPos = i + 1; } state = 'code'; } @@ -453,72 +742,58 @@ function scanForImports( if (shouldStrip) { anyCommentStripped = true; - // Find the end of the comment and check what's after + // Check what follows the comment on its line: its first non-whitespace + // character, and whether the rest of the line is blank const afterCommentPos = i + 2; - let afterCommentContent = ''; - let nextNewlinePos = sourceCode.indexOf('\n', afterCommentPos); - if (nextNewlinePos === -1) { - nextNewlinePos = sourceCode.length; + let afterCommentStart = afterCommentPos; + while ( + afterCommentStart < len && + sourceCode[afterCommentStart] !== '\n' && + isWhitespace(sourceCode[afterCommentStart]) + ) { + afterCommentStart += 1; } - afterCommentContent = sourceCode.slice(afterCommentPos, nextNewlinePos); // Check for JSX comment syntax: {/* comment */} - // preCommentContent ends with '{' (ignoring whitespace) and afterCommentContent starts with '}' (ignoring whitespace) - const trimmedPreComment = preCommentContent.trimEnd(); - const trimmedAfterComment = afterCommentContent.trimStart(); + // preCommentContent ends with '{' (ignoring whitespace) and the comment is followed by '}' (ignoring whitespace) const isJsxComment = - trimmedPreComment.endsWith('{') && trimmedAfterComment.startsWith('}'); + sourceCode[afterCommentStart] === '}' && preCommentContent.endsWith('{'); // For JSX comments, check if removing the braces leaves only whitespace const preCommentWithoutBrace = isJsxComment - ? trimmedPreComment.slice(0, -1) + ? preCommentContent.slice(0, -1).trimEnd() : preCommentContent; - const afterCommentWithoutBrace = isJsxComment - ? trimmedAfterComment.slice(1) - : afterCommentContent; + const afterCommentIsBlank = isJsxComment + ? isRestOfLineBlank(sourceCode, afterCommentStart + 1) + : isRestOfLineBlank(sourceCode, afterCommentPos); - const isCommentOnlyLines = - preCommentWithoutBrace.trim() === '' && afterCommentWithoutBrace.trim() === ''; + const isCommentOnlyLines = preCommentWithoutBrace === '' && afterCommentIsBlank; if (isCommentOnlyLines) { // Skip the entire comment and everything up to and including the next newline // For JSX comments, this also skips the surrounding braces - i = nextNewlinePos; - if (i < len && sourceCode[i] === '\n') { - // Skip the newline entirely - advance to the character after it - i += 1; - lineStartPos = i; - } else { - lineStartPos = i; - } + const nextNewlinePos = sourceCode.indexOf('\n', afterCommentPos); + i = nextNewlinePos === -1 ? len : nextNewlinePos + 1; state = 'code'; preCommentContent = ''; continue; } else if (isJsxComment) { // JSX comment is inline with other code - strip the braces too // e.g., `