fix(portal): name the .md pages whose HTML and components the portal leaves out

src/prompts/portal/source.ts

@@ -3,6 +3,7 @@ import { APIMATIC_CONFIG_FILE_NAME } from '../../types/apimatic-config/document.
 import { DirectoryPath } from '../../types/file/directoryPath.js';
 import { listedInProse } from '../../utils/string-utils.js';
 import { ContentNotices, NoTabsListed } from '../../types/portal/content-notices.js';
+import { LeftOutHtml } from '../../types/portal/left-out-html.js';
 import { API_REFERENCE_TOKEN, NAVIGATION_FILE_NAME, TabEntry } from '../../types/portal/portal-navigation.js';
 import {
   ContentProblem,
@@ -430,6 +431,7 @@ export function reportIgnoredNavigationFiles(files: FilePath[], sourceDirectory:
 }
 
 export function reportContentNotices(notices: ContentNotices, sourceDirectory: DirectoryPath): void {
+  reportLeftOutHtml(notices.leftOutHtml, sourceDirectory);
   reportHiddenPages(notices.hiddenPages, sourceDirectory);
   reportIgnoredNavigationFiles(notices.ignoredNavigationFiles, sourceDirectory);
   reportSharedTabNames(notices.sharedTabNames, sourceDirectory);
@@ -513,6 +515,28 @@ export function reportSharedTabNames(shared: SharedTabName[], sourceDirectory: D
   );
 }
 
+export function reportLeftOutHtml(pages: LeftOutHtml[], sourceDirectory: DirectoryPath): void {
+  if (pages.length === 0) {
+    return;
+  }
+  const one = pages.length === 1;
+  const lines = pages.flatMap((html) => {
+    const page = relative(html.page, sourceDirectory);
+    const shown = html.shown().map(({ line, text }) => `  • ${page}, line ${line}: ${f.var(text)}`);
+    const unshown = html.unshown();
+    return unshown === 0 ? shown : [...shown, `    and ${unshown} more on this page`];
+  });
+  const holding = one
+    ? `A ${f.var('.md')} page in ${f.path(sourceDirectory)} has`
+    : `Some ${f.var('.md')} pages in ${f.path(sourceDirectory)} have`;
+  log.warn(`${holding} HTML or components, which the portal leaves out:`);
+  log.message(lines.join('\n'));
+  log.message(
+    `Rename ${one ? 'the page' : 'each page'} to ${f.var('.mdx')} to keep them. MDX needs empty tags closed, ` +
+      `such as ${f.var('<br />')}, and comments written as ${f.var('{/* … */}')}.`
+  );
+}
+
 export function reportHiddenPages(files: FilePath[], sourceDirectory: DirectoryPath): void {
   if (files.length === 0) {
     return;

src/types/portal/content-notices.ts

@@ -1,4 +1,5 @@
 import { FilePath } from '../file/filePath.js';
+import { LeftOutHtml } from './left-out-html.js';
 import { TabEntry } from './portal-navigation.js';
 import { SharedTabName } from './portal-tabs.js';
 
@@ -11,6 +12,8 @@ export interface NoTabsListed {
 
 /** What a build accepts in `content/`, but the user should hear of. */
 export interface ContentNotices {
+  /** Warned of rather than refused: the rest of the page builds, and one stray `<br>` should not stop a build. */
+  leftOutHtml: LeftOutHtml[];
   /**
    * Pages below `content/api/<slug>/` for a specification `<slug>`. The section's generated
    * metadata lists only the reference pages, so these never appear in the sidebar.

src/types/portal/content-tree.ts

@@ -6,6 +6,7 @@ import { FileName } from '../file/fileName.js';
 import { FilePath } from '../file/filePath.js';
 import { ContentNotices } from './content-notices.js';
 import { GENERATED_SECTIONS, GeneratedPages } from './generated-pages.js';
+import { LeftOutHtml } from './left-out-html.js';
 import { ParsedPage } from './page.js';
 import {
   API_REFERENCE_NAME,
@@ -190,6 +191,9 @@ export class ContentTree {
     const rootNavigation = navigation.root.navigation;
     const unplaced = ContentTree.unplaced(rootNavigation?.settings, generatedPages);
     return ok({
+      leftOutHtml: read.pages.flatMap(({ file, parsed }) =>
+        parsed === undefined || parsed.leftOutHtml.length === 0 ? [] : [new LeftOutHtml(file, parsed.leftOutHtml)]
+      ),
       hiddenPages: ContentTree.hiddenPages(pages, specs),
       ignoredNavigationFiles: navigation.ignoredFiles,
       sharedTabNames: sharedTabNames(tabs),

src/types/portal/left-out-html.ts

@@ -0,0 +1,37 @@
+import { FilePath } from '../file/filePath.js';
+import { HtmlPiece } from './page.js';
+
+// Renaming the page keeps all of it, so a few pieces say enough.
+const SHOWN = 3;
+// Enough to find a piece by, where its line can run to a whole paragraph.
+const SHOWN_LENGTH = 60;
+
+/** The HTML a `.md` page holds, which the build leaves out: a `.md` page is Markdown without HTML or components. */
+export class LeftOutHtml {
+  constructor(public readonly page: FilePath, private readonly pieces: HtmlPiece[]) {}
+
+  public shown(): HtmlPiece[] {
+    return this.distinct()
+      .slice(0, SHOWN)
+      .map(({ line, text }) => ({
+        line,
+        text: text.length > SHOWN_LENGTH ? `${text.slice(0, SHOWN_LENGTH - 1)}…` : text
+      }));
+  }
+
+  public unshown(): number {
+    return Math.max(this.distinct().length - SHOWN, 0);
+  }
+
+  // By what is written, not where, or every edit above a piece would give the notice again.
+  public addsNothingTo(earlier: LeftOutHtml): boolean {
+    return (
+      this.page.isEqual(earlier.page) &&
+      this.pieces.every(({ text }) => earlier.pieces.some((piece) => piece.text === text))
+    );
+  }
+
+  private distinct(): HtmlPiece[] {
+    return this.pieces.filter((piece, index) => this.pieces.findIndex(({ text }) => text === piece.text) === index);
+  }
+}

src/types/portal/page.ts

@@ -22,10 +22,18 @@ export interface PageImage {
   path: string;
 }
 
-/** A page as the build parses it: its front matter held to the build's schema, and the images the build imports. */
+/** A piece of HTML in a `.md` page, by the line it starts on and that line as written. */
+export interface HtmlPiece {
+  line: number;
+  text: string;
+}
+
+/** A page as the build parses it. */
 export interface ParsedPage {
   frontMatter: Result<PageFrontMatter, string[]>;
   images: PageImage[];
+  /** None in an `.mdx` page, which renders its HTML. */
+  leftOutHtml: HtmlPiece[];
 }
 
 type SchemaIssue = NonNullable<ReturnType<typeof pageSchema.safeParse>['error']>['issues'][number];
@@ -38,27 +46,40 @@ interface MarkdownNode {
   identifier?: string;
   name?: string | null;
   attributes?: { type: string; name?: string; value?: unknown }[];
+  value?: string;
   position?: { start: { line: number } };
   children?: MarkdownNode[];
 }
 
 // A scheme, a drive letter or a network path: none of them names a file the build imports.
 const NOT_A_FILE = /^(?:[a-z][a-z\d+.-]*:|\/\/)/i;
 
+// As CommonMark reads one, where `<!-->` and `<!--->` are whole comments.
+const COMMENT = /<!--(?:-?>|[\s\S]*?(?:-->|$))/g;
+const CLOSING_TAG = /<\/[a-z][a-z\d-]*\s*>/gi;
+
 /** Parsed once, as the build parses it, `.mdx` as MDX; a body that does not parse is the build's to report. */
 export async function parsePage(contents: string, fileName: FileName, label: string): Promise<ParsedPage> {
   let parsed: { matter: string; data: unknown };
   try {
     parsed = frontmatter(contents);
   } catch (error) {
     const reason = errorMessage(error).split('\n')[0];
-    return { frontMatter: err([`${label}: its front matter is not valid YAML. ${reason}`]), images: [] };
+    return {
+      frontMatter: err([`${label}: its front matter is not valid YAML. ${reason}`]),
+      images: [],
+      leftOutHtml: []
+    };
   }
   // Blanked rather than cut, so every line keeps its number.
   const body = '\n'.repeat(parsed.matter.split('\n').length - 1) + contents.slice(parsed.matter.length);
+  const isMdx = fileName.hasExactExtension('.mdx');
+  const tree = await parsedBody(body, isMdx);
+  const nodes = tree === null ? [] : nodesOf(tree);
   return {
     frontMatter: await checkedFrontMatter(parsed, label),
-    images: await imagesIn(body, fileName.hasExactExtension('.mdx'))
+    images: imagesIn(nodes),
+    leftOutHtml: isMdx ? [] : htmlIn(nodes)
   };
 }
 
@@ -106,39 +127,43 @@ function titleError(title: unknown, label: string): string {
   return `${label}: 'title' must be text. Put it in quotes if it looks like a number or true or false.`;
 }
 
-async function imagesIn(body: string, isMdx: boolean): Promise<PageImage[]> {
+async function parsedBody(body: string, isMdx: boolean): Promise<MarkdownNode | null> {
   // On first use, as the schema is: most commands never read a page.
   const { remark } = await import('remark');
   const processor = isMdx ? remark().use((await import('remark-mdx')).default) : remark();
-  let tree: MarkdownNode;
   try {
-    tree = processor.parse(body) as MarkdownNode;
+    return processor.parse(body) as MarkdownNode;
   } catch {
-    return [];
+    return null;
   }
+}
+
+function nodesOf(node: MarkdownNode): MarkdownNode[] {
+  return [node, ...(node.children ?? []).flatMap(nodesOf)];
+}
 
+function imagesIn(nodes: MarkdownNode[]): PageImage[] {
   // The template writes `![a][ref]` as `![a](url)` before the build imports it (remark-image-references.ts).
   const definitions = new Map<string, string>();
-  const define = (node: MarkdownNode) => {
+  for (const node of nodes) {
     // The first definition of a label wins, as CommonMark reads them.
     if (node.type === 'definition' && node.identifier !== undefined && !definitions.has(node.identifier)) {
       definitions.set(node.identifier, node.url ?? '');
     }
-    node.children?.forEach(define);
-  };
-  define(tree);
-
-  const images: PageImage[] = [];
-  const visit = (node: MarkdownNode) => {
+  }
+  return nodes.flatMap((node) => {
     const url = addressOf(node, definitions);
     const image = url !== undefined ? located(url) : null;
-    if (image !== null) {
-      images.push({ ...image, line: node.position?.start.line ?? 0 });
-    }
-    node.children?.forEach(visit);
-  };
-  visit(tree);
-  return images;
+    return image === null ? [] : [{ ...image, line: node.position?.start.line ?? 0 }];
+  });
+}
+
+function htmlIn(nodes: MarkdownNode[]): HtmlPiece[] {
+  // Comments and closing tags leave out nothing a reader would see.
+  const shows = (html: string) => html.replace(COMMENT, '').replace(CLOSING_TAG, '').trim() !== '';
+  return nodes.flatMap(({ type, value = '', position }) =>
+    type === 'html' && shows(value) ? [{ line: position?.start.line ?? 0, text: value.split('\n')[0].trim() }] : []
+  );
 }
 
 function addressOf(node: MarkdownNode, definitions: Map<string, string>): string | undefined {

src/types/portal/preview-content.ts

@@ -33,6 +33,9 @@ export class PreviewContent {
 
   private static since(current: ContentNotices, previous: ContentNotices): ContentNotices {
     return {
+      leftOutHtml: whenAnyNew(current.leftOutHtml, previous.leftOutHtml, (html, earlier) =>
+        html.addsNothingTo(earlier)
+      ),
       hiddenPages: whenAnyNew(current.hiddenPages, previous.hiddenPages, (left, right) => left.isEqual(right)),
       ignoredNavigationFiles: whenAnyNew(
         current.ignoredNavigationFiles,

test/infrastructure/portal-project-service.test.ts

@@ -68,6 +68,7 @@ describe('PortalProjectService', () => {
     staticDirectory: null,
     shadowedFiles: [],
     contentNotices: {
+      leftOutHtml: [],
       hiddenPages: [],
       ignoredNavigationFiles: [],
       sharedTabNames: [],

test/prompts/portal/serve.test.ts

@@ -200,6 +200,7 @@ describe('PortalServePrompts', () => {
 
   describe('a save the preview accepts', () => {
     const NO_NOTICES: ContentNotices = {
+      leftOutHtml: [],
       hiddenPages: [],
       ignoredNavigationFiles: [],
       sharedTabNames: [],

test/prompts/portal/source.test.ts

@@ -4,6 +4,7 @@ import sinon from 'sinon';
 import { log } from '@clack/prompts';
 import { PortalServePrompts } from '../../../src/prompts/portal/serve.js';
 import {
+  reportLeftOutHtml,
   reportLeftOutSpecs,
   reportNoTabsListed,
   reportShadowedFiles,
@@ -17,8 +18,20 @@ import { DirectoryPath } from '../../../src/types/file/directoryPath.js';
 import { FileName } from '../../../src/types/file/fileName.js';
 import { FilePath } from '../../../src/types/file/filePath.js';
 import { PLUGIN_SECTION, SDK_SECTION } from '../../../src/types/portal/generated-pages.js';
+import { LeftOutHtml } from '../../../src/types/portal/left-out-html.js';
 import { OAUTH_CALLBACK } from '../../../src/types/portal/content-tree.js';
 
+/** What `log.warn` and `log.message` print from here on, without colours, until `sinon.restore`. */
+function recordedLines(): string[] {
+  const lines: string[] = [];
+  const record = (text?: string | string[]) => {
+    lines.push(stripVTControlCharacters(String(text)));
+  };
+  sinon.stub(log, 'warn').callsFake(record);
+  sinon.stub(log, 'message').callsFake(record);
+  return lines;
+}
+
 describe('reportSourceProblem', () => {
   const source = new DirectoryPath('project').join('src');
   let error: sinon.SinonStub;
@@ -592,12 +605,7 @@ describe('reportSharedTabNames', () => {
   const printed = () => lines.join('\n');
 
   beforeEach(() => {
-    lines = [];
-    const record = (text?: string | string[]) => {
-      lines.push(stripVTControlCharacters(String(text)));
-    };
-    sinon.stub(log, 'warn').callsFake(record);
-    sinon.stub(log, 'message').callsFake(record);
+    lines = recordedLines();
   });
 
   afterEach(() => {
@@ -853,3 +861,65 @@ describe('reportLeftOutSpecs', () => {
     ]);
   });
 });
+
+describe('reportLeftOutHtml', () => {
+  const source = new DirectoryPath('project').join('src');
+  const index = new FilePath(source.join('content'), new FileName('index.md'));
+  const intro = new FilePath(source.join('content', 'guides'), new FileName('intro.md'));
+  let lines: string[];
+
+  beforeEach(() => {
+    lines = recordedLines();
+  });
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('says nothing when no .md page has HTML', () => {
+    reportLeftOutHtml([], source);
+
+    expect(lines).to.deep.equal([]);
+  });
+
+  it('names the HTML of a page with its lines, and how to keep it', () => {
+    reportLeftOutHtml(
+      [
+        new LeftOutHtml(index, [
+          { line: 5, text: '<br>' },
+          { line: 9, text: '<Callout type="warn">' }
+        ])
+      ],
+      source
+    );
+
+    expect(lines.join('\n').split('\n')).to.deep.equal([
+      `A '.md' page in '${source.toString()}' has HTML or components, which the portal leaves out:`,
+      "  • 'content/index.md', line 5: '<br>'",
+      "  • 'content/index.md', line 9: '<Callout type=\"warn\">'",
+      "Rename the page to '.mdx' to keep them. MDX needs empty tags closed, such as '<br />', and comments written " +
+        "as '{/* … */}'."
+    ]);
+  });
+
+  it('counts what a page does not show', () => {
+    const pieces = ['<br>', '<details>', '<span>', '<kbd>', '<i>'].map((text, index) => ({ line: index + 1, text }));
+
+    reportLeftOutHtml([new LeftOutHtml(index, pieces)], source);
+
+    expect(lines[1].split('\n').at(-1)).to.equal('    and 2 more on this page');
+  });
+
+  it('speaks of each page when there are several', () => {
+    reportLeftOutHtml(
+      [new LeftOutHtml(index, [{ line: 5, text: '<br>' }]), new LeftOutHtml(intro, [{ line: 3, text: '<Cards>' }])],
+      source
+    );
+
+    expect(lines[0]).to.equal(
+      `Some '.md' pages in '${source.toString()}' have HTML or components, which the portal leaves out:`
+    );
+    expect(lines[1]).to.contain("'content/guides/intro.md', line 3: '<Cards>'");
+    expect(lines[2]).to.match(/^Rename each page to '\.mdx'/);
+  });
+});