diff --git a/.github/workflows/ci-website-build.yaml b/.github/workflows/ci-website-build.yaml
index cd6bd6c3b86..0a86fcbfd19 100644
--- a/.github/workflows/ci-website-build.yaml
+++ b/.github/workflows/ci-website-build.yaml
@@ -50,6 +50,9 @@ jobs:
- name: Validate llms.txt links
run: pnpm --filter @comfyorg/website validate:llms-txt-links
+ - name: Validate advertised markdown copies
+ run: pnpm --filter @comfyorg/website validate:markdown-alternates
+
- name: Check internal links end with a slash
run: pnpm --filter @comfyorg/website check:link-slashes
@@ -77,6 +80,7 @@ jobs:
fi
pnpm --filter @comfyorg/website build
pnpm --filter @comfyorg/website check:link-slashes
+ pnpm --filter @comfyorg/website validate:markdown-alternates
if [ -d apps/website/dist/workshop ]; then
echo "::error title=Workshop leaked into a release build::apps/website/dist/workshop exists with WORKSHOP_IN_BUILD=0. The retired Workshop routes must not deploy. See apps/website/src/integrations/workshop-release-gate.ts."
exit 1
diff --git a/apps/website/package.json b/apps/website/package.json
index 19e27050a2c..14669a8e9dd 100644
--- a/apps/website/package.json
+++ b/apps/website/package.json
@@ -44,6 +44,7 @@
"check:models-html": "tsx ./scripts/check-models-html.ts",
"list:models-pages": "tsx ./scripts/list-models-pages.ts",
"validate:llms-txt-links": "tsx ./scripts/validate-llms-txt-links.ts",
+ "validate:markdown-alternates": "tsx ./scripts/validate-markdown-alternates.ts",
"indexnow:submit": "tsx ./scripts/indexnow-submit.ts",
"check:link-slashes": "tsx ./scripts/check-internal-link-slashes.ts",
"workshop:generate": "tsx ./scripts/generate-workshop-models.ts",
diff --git a/apps/website/scripts/check-hreflang.ts b/apps/website/scripts/check-hreflang.ts
index 2bcf9d41bbc..8819a2449cf 100644
--- a/apps/website/scripts/check-hreflang.ts
+++ b/apps/website/scripts/check-hreflang.ts
@@ -10,8 +10,8 @@
*/
import type { Alternate } from '@/utils/hreflangRoutes'
-import { existsSync, readdirSync, readFileSync } from 'node:fs'
-import { join, relative, sep } from 'node:path'
+import { existsSync, readFileSync } from 'node:fs'
+import { join, relative } from 'node:path'
import {
auditBuiltSite,
@@ -19,25 +19,12 @@ import {
sitemapChunkNames
} from '@/utils/hreflangAudit'
+import { htmlFiles, routeOf } from './dist-html'
+
const DIST = join(process.cwd(), 'dist')
const PUBLIC = join(process.cwd(), 'public')
const ORIGIN = 'https://comfy.org'
-function htmlFiles(dir: string): string[] {
- return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
- const full = join(dir, entry.name)
- if (entry.isDirectory()) return htmlFiles(full)
- return entry.name.endsWith('.html') ? [full] : []
- })
-}
-
-/** `dist/zh-CN/about/index.html` -> `/zh-CN/about/` */
-function routeOf(file: string): string {
- const rel = relative(DIST, file).split(sep).join('/')
- const withoutIndex = rel.replace(/index\.html$/, '')
- return `/${withoutIndex}`.replace(/\/{2,}/g, '/')
-}
-
function alternatesIn(html: string): Alternate[] {
const out: Alternate[] = []
const re = /()
const canonicals = new Map()
for (const file of files) {
- const route = routeOf(file)
+ const route = routeOf(DIST, file)
const html = readFileSync(file, 'utf-8')
const alternates = alternatesIn(html)
pages.set(route, alternates)
diff --git a/apps/website/scripts/check-internal-link-slashes.ts b/apps/website/scripts/check-internal-link-slashes.ts
index 1b8eb10878b..a0913c5b554 100644
--- a/apps/website/scripts/check-internal-link-slashes.ts
+++ b/apps/website/scripts/check-internal-link-slashes.ts
@@ -3,23 +3,17 @@
* slash, or through the www host. Both reach the page only via a second
* address, which splits its signals until a crawler settles on the canonical.
*/
-import { existsSync, readdirSync, readFileSync } from 'node:fs'
+import { existsSync, readFileSync } from 'node:fs'
import { join, relative, sep } from 'node:path'
import { aliasHostHrefs, slashlessPageHrefs } from '@/utils/internalLinkSlashes'
+import { htmlFiles } from './dist-html'
+
const DIST = join(process.cwd(), 'dist')
const ORIGINS = ['https://comfy.org']
const ALIAS_ORIGINS = ['https://www.comfy.org']
-function htmlFiles(dir: string): string[] {
- return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
- const full = join(dir, entry.name)
- if (entry.isDirectory()) return htmlFiles(full)
- return entry.name.endsWith('.html') ? [full] : []
- })
-}
-
if (!existsSync(DIST)) {
console.error(`[link-slashes] ${DIST} does not exist; build first.`)
process.exit(1)
diff --git a/apps/website/scripts/dist-html.test.ts b/apps/website/scripts/dist-html.test.ts
new file mode 100644
index 00000000000..620609357e8
--- /dev/null
+++ b/apps/website/scripts/dist-html.test.ts
@@ -0,0 +1,42 @@
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { beforeAll, afterAll, describe, expect, it } from 'vitest'
+
+import { htmlFiles, isFileWithExactCase, routeOf } from './dist-html'
+
+let root: string
+
+beforeAll(async () => {
+ root = await mkdtemp(join(tmpdir(), 'dist-html-'))
+ await mkdir(join(root, 'zh-CN', 'about'), { recursive: true })
+ await writeFile(join(root, 'index.html'), '')
+ await writeFile(join(root, 'searchindex.html'), '')
+ await writeFile(join(root, 'zh-CN', 'about', 'index.html'), '')
+ await writeFile(join(root, 'zh-CN', 'cli.md'), '')
+})
+
+afterAll(() => rm(root, { recursive: true, force: true }))
+
+describe('htmlFiles and routeOf', () => {
+ it('lists every built page as its route', () => {
+ expect(
+ htmlFiles(root)
+ .map((file) => routeOf(root, file))
+ .sort()
+ ).toEqual(['/', '/searchindex.html', '/zh-CN/about/'])
+ })
+})
+
+describe('isFileWithExactCase', () => {
+ it.for([
+ { path: '/zh-CN/cli.md', exists: true },
+ { path: '/zh-CN/CLI.md', exists: false },
+ { path: '/ZH-cn/cli.md', exists: false },
+ { path: '/zh-CN/missing.md', exists: false },
+ { path: '/zh-CN', exists: false },
+ { path: '/', exists: false }
+ ])('$path exists: $exists', ({ path, exists }) => {
+ expect(isFileWithExactCase(root, path)).toBe(exists)
+ })
+})
diff --git a/apps/website/scripts/dist-html.ts b/apps/website/scripts/dist-html.ts
new file mode 100644
index 00000000000..13a2b15cc36
--- /dev/null
+++ b/apps/website/scripts/dist-html.ts
@@ -0,0 +1,38 @@
+import { readdirSync } from 'node:fs'
+import { join, relative, sep } from 'node:path'
+
+/** Every `.html` file under `dir`, recursively. Throws when `dir` is missing. */
+export function htmlFiles(dir: string): string[] {
+ return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+ const full = join(dir, entry.name)
+ if (entry.isDirectory()) return htmlFiles(full)
+ return entry.name.endsWith('.html') ? [full] : []
+ })
+}
+
+/** `/zh-CN/about/index.html` -> `/zh-CN/about/` */
+export function routeOf(root: string, file: string): string {
+ const rel = relative(root, file).split(sep).join('/')
+ const withoutIndex = rel.replace(/(^|\/)index\.html$/, '$1')
+ return `/${withoutIndex}`.replace(/\/{2,}/g, '/')
+}
+
+/**
+ * Whether `path` (`/zh-CN/cli.md`) names a file under `root` with this exact
+ * case. `existsSync` ignores case on macOS and Windows; the Linux CI and the
+ * CDN do not.
+ */
+export function isFileWithExactCase(root: string, path: string): boolean {
+ const segments = path.split('/').filter(Boolean)
+ return (
+ segments.length > 0 &&
+ segments.every((segment, index) => {
+ const entry = readdirSync(join(root, ...segments.slice(0, index)), {
+ withFileTypes: true
+ }).find(({ name }) => name === segment)
+ return index === segments.length - 1
+ ? entry?.isFile() === true
+ : entry?.isDirectory() === true
+ })
+ )
+}
diff --git a/apps/website/scripts/validate-markdown-alternates.ts b/apps/website/scripts/validate-markdown-alternates.ts
new file mode 100644
index 00000000000..a6b00e062ff
--- /dev/null
+++ b/apps/website/scripts/validate-markdown-alternates.ts
@@ -0,0 +1,57 @@
+/**
+ * Fails when too few built pages advertise a markdown copy, or a page
+ * advertises one that was not built or whose front matter does not name that
+ * page as its canonical.
+ */
+import { existsSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import {
+ advertisementShortfall,
+ auditMarkdownAlternates
+} from '@/utils/markdownAlternateAudit'
+
+import { htmlFiles, isFileWithExactCase, routeOf } from './dist-html'
+
+const DIST = join(process.cwd(), 'dist')
+const ORIGIN = 'https://comfy.org'
+
+function readTwin(path: string): string | undefined {
+ return isFileWithExactCase(DIST, path)
+ ? readFileSync(join(DIST, path), 'utf-8')
+ : undefined
+}
+
+if (!existsSync(DIST)) {
+ console.error(`[markdown-alternates] ${DIST} does not exist; build first.`)
+ process.exit(1)
+}
+
+const pages = htmlFiles(DIST).map((file) => ({
+ route: routeOf(DIST, file),
+ html: readFileSync(file, 'utf-8')
+}))
+const { advertised, canonicalChecked, problems } = auditMarkdownAlternates(
+ pages,
+ readTwin,
+ ORIGIN
+)
+
+const shortfall = advertisementShortfall(advertised, pages.length)
+if (shortfall !== undefined) {
+ console.error(`[markdown-alternates] ${shortfall}`)
+ process.exit(1)
+}
+
+if (problems.length > 0) {
+ console.error(
+ `[markdown-alternates] ${problems.length} of ${advertised} advertised markdown copies are broken:`
+ )
+ for (const problem of problems) console.error(` ${problem}`)
+ process.exit(1)
+}
+
+console.warn(
+ `[markdown-alternates] ${advertised} of ${pages.length} pages advertise a markdown copy, and every copy exists. ` +
+ `${canonicalChecked} name their page as canonical; ${advertised - canonicalChecked} pages emit no canonical to compare.`
+)
diff --git a/apps/website/src/utils/markdownAlternateAudit.test.ts b/apps/website/src/utils/markdownAlternateAudit.test.ts
new file mode 100644
index 00000000000..357d5b671a3
--- /dev/null
+++ b/apps/website/src/utils/markdownAlternateAudit.test.ts
@@ -0,0 +1,216 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+ advertisementShortfall,
+ auditMarkdownAlternates
+} from './markdownAlternateAudit'
+
+const ORIGIN = 'https://comfy.org'
+
+function page(route: string, twinHref?: string, canonical?: string) {
+ return pageWithLink(
+ route,
+ twinHref &&
+ ``,
+ canonical
+ )
+}
+
+function pageWithLink(route: string, twinLink?: string, canonical?: string) {
+ const head = [
+ canonical && ``,
+ twinLink
+ ]
+ return {
+ route,
+ html: `${head.filter(Boolean).join('')}`
+ }
+}
+
+function twin(canonical?: string) {
+ const front = canonical
+ ? `title: "T"\ncanonical: ${canonical}\n`
+ : 'title: "T"\n'
+ return `---\n${front}---\n\n# T\n`
+}
+
+describe('auditMarkdownAlternates', () => {
+ it.for([
+ {
+ name: 'a twin that names its page',
+ twins: { '/cli.md': twin(`${ORIGIN}/cli/`) },
+ canonicalChecked: 1,
+ problems: []
+ },
+ {
+ name: 'a twin that was not built',
+ twins: {},
+ canonicalChecked: 0,
+ problems: ['/cli/: advertises /cli.md, which was not built']
+ },
+ {
+ name: 'a twin without a canonical',
+ twins: { '/cli.md': twin() },
+ canonicalChecked: 0,
+ problems: ['/cli/: /cli.md has no canonical in its front matter']
+ },
+ {
+ name: 'a twin whose canonical drops the trailing slash',
+ twins: { '/cli.md': twin(`${ORIGIN}/cli`) },
+ canonicalChecked: 1,
+ problems: [
+ `/cli/: /cli.md names canonical ${ORIGIN}/cli, the page says ${ORIGIN}/cli/`
+ ]
+ }
+ ])('reports $name', ({ twins, canonicalChecked, problems }) => {
+ const report = auditMarkdownAlternates(
+ [page('/cli/', '/cli.md', `${ORIGIN}/cli/`)],
+ (path) => twins[path as keyof typeof twins],
+ ORIGIN
+ )
+ expect(report).toEqual({ advertised: 1, canonicalChecked, problems })
+ })
+
+ it.for([
+ ``,
+ '',
+ '',
+ ''
+ ])('reads the href of %s', (link) => {
+ const report = auditMarkdownAlternates(
+ [pageWithLink('/cli/', link)],
+ () => undefined,
+ ORIGIN
+ )
+ expect(report.problems).toEqual([
+ '/cli/: advertises /cli.md, which was not built'
+ ])
+ })
+
+ it.for([
+ {
+ name: 'a preload link',
+ head: ''
+ },
+ {
+ name: 'a link inside a comment',
+ head: ''
+ },
+ {
+ name: 'a link inside a script',
+ head: ''
+ }
+ ])('does not count $name as an advertisement', ({ head }) => {
+ const report = auditMarkdownAlternates(
+ [{ route: '/cli/', html: `${head}` }],
+ () => undefined,
+ ORIGIN
+ )
+ expect(report).toEqual({ advertised: 0, canonicalChecked: 0, problems: [] })
+ })
+
+ it('reads the real link after a commented-out one', () => {
+ const report = auditMarkdownAlternates(
+ [
+ pageWithLink(
+ '/cli/',
+ '' +
+ ''
+ )
+ ],
+ (path) => (path === '/cli.md' ? twin(`${ORIGIN}/cli/`) : undefined),
+ ORIGIN
+ )
+ expect(report.problems).toEqual([
+ '/cli/: advertises /missing.md, which was not built'
+ ])
+ })
+
+ it('reports a twin href on another origin', () => {
+ const report = auditMarkdownAlternates(
+ [page('/cli/', 'https://other.example/cli.md')],
+ () => twin(`${ORIGIN}/cli/`),
+ ORIGIN
+ )
+ expect(report.problems).toEqual([
+ `/cli/: advertises https://other.example/cli.md, which is not on ${ORIGIN}`
+ ])
+ })
+
+ it('reports a markdown link whose href cannot be read', () => {
+ const report = auditMarkdownAlternates(
+ [pageWithLink('/cli/', '')],
+ () => twin(`${ORIGIN}/cli/`),
+ ORIGIN
+ )
+ expect(report).toEqual({
+ advertised: 1,
+ canonicalChecked: 0,
+ problems: ['/cli/: has a text/markdown link with no readable href']
+ })
+ })
+
+ it('reports an href that is not a valid URL instead of throwing', () => {
+ const report = auditMarkdownAlternates(
+ [page('/guide/', '/100%-guide.md'), page('/cli/', '/cli.md')],
+ (path) => (path === '/cli.md' ? twin(`${ORIGIN}/cli/`) : undefined),
+ ORIGIN
+ )
+ expect(report.problems).toEqual([
+ '/guide/: advertises /100%-guide.md, which is not a valid URL'
+ ])
+ })
+
+ it('skips pages that advertise no markdown copy', () => {
+ const report = auditMarkdownAlternates(
+ [page('/payment/success/')],
+ () => undefined,
+ ORIGIN
+ )
+ expect(report).toEqual({ advertised: 0, canonicalChecked: 0, problems: [] })
+ })
+
+ it('checks only existence when the page emits no canonical', () => {
+ const report = auditMarkdownAlternates(
+ [page('/old-cli/', '/cli.md')],
+ (path) => (path === '/cli.md' ? twin(`${ORIGIN}/cli/`) : undefined),
+ ORIGIN
+ )
+ expect(report).toEqual({ advertised: 1, canonicalChecked: 0, problems: [] })
+ })
+
+ it('resolves a percent-encoded twin href to the built file name', () => {
+ const report = auditMarkdownAlternates(
+ [
+ page(
+ '/zh-CN/café/',
+ '/zh-CN/caf%C3%A9.md',
+ `${ORIGIN}/zh-CN/caf%C3%A9/`
+ )
+ ],
+ (path) =>
+ path === '/zh-CN/café.md'
+ ? twin(`${ORIGIN}/zh-CN/caf%C3%A9/`)
+ : undefined,
+ ORIGIN
+ )
+ expect(report).toEqual({ advertised: 1, canonicalChecked: 1, problems: [] })
+ })
+})
+
+describe('advertisementShortfall', () => {
+ it.for([
+ { advertised: 855, builtPages: 950, fails: false },
+ { advertised: 760, builtPages: 950, fails: false },
+ { advertised: 759, builtPages: 950, fails: true },
+ { advertised: 55, builtPages: 950, fails: true },
+ { advertised: 0, builtPages: 0, fails: true }
+ ])(
+ '$advertised of $builtPages advertising fails: $fails',
+ ({ advertised, builtPages, fails }) => {
+ expect(advertisementShortfall(advertised, builtPages) !== undefined).toBe(
+ fails
+ )
+ }
+ )
+})
diff --git a/apps/website/src/utils/markdownAlternateAudit.ts b/apps/website/src/utils/markdownAlternateAudit.ts
new file mode 100644
index 00000000000..4b6d0c2207b
--- /dev/null
+++ b/apps/website/src/utils/markdownAlternateAudit.ts
@@ -0,0 +1,160 @@
+/**
+ * The rule `scripts/validate-markdown-alternates.ts` enforces against dist/:
+ * every markdown copy a page advertises exists, and names that page as its
+ * canonical, so agents never follow our link into a 404 and search engines
+ * never treat the copy as a second page.
+ */
+import { parse } from 'parse5'
+import type { DefaultTreeAdapterTypes } from 'parse5'
+
+export interface BuiltPage {
+ /** `/cli/`, `/zh-CN/about/`, `/` */
+ route: string
+ html: string
+}
+
+export interface MarkdownAlternateReport {
+ advertised: number
+ /** Advertising pages with a `rel="canonical"` to compare the twin's against. */
+ canonicalChecked: number
+ problems: string[]
+}
+
+/**
+ * 855 of 950 built pages (90%) advertise a copy today; the rest are noindex,
+ * redirect or empty pages. 80% leaves room for ~95 more of those, and fails
+ * when any large family (149 /hub/models, 158 zh-CN, 383 supported-models)
+ * loses its link.
+ */
+const MIN_ADVERTISED_SHARE = 0.8
+
+const FRONT_MATTER = /^---\n([\s\S]*?)\n---(?:\n|$)/
+const CANONICAL_FIELD = /^canonical:\s*(\S+)\s*$/m
+
+type Attributes = Map
+
+function linkElements(node: DefaultTreeAdapterTypes.ParentNode): Attributes[] {
+ return node.childNodes.flatMap((child) => {
+ if (!('tagName' in child)) return []
+ const nested = linkElements(child)
+ if (child.tagName !== 'link') return nested
+ const attributes = new Map(
+ child.attrs.map(({ name, value }) => [name, value])
+ )
+ return [attributes, ...nested]
+ })
+}
+
+function hasRel(link: Attributes, rel: string): boolean {
+ return (link.get('rel') ?? '').toLowerCase().split(/\s+/).includes(rel)
+}
+
+function isMarkdownAlternate(link: Attributes): boolean {
+ return (
+ hasRel(link, 'alternate') &&
+ link.get('type')?.trim().toLowerCase() === 'text/markdown'
+ )
+}
+
+function twinCanonical(markdown: string): string | undefined {
+ const front = FRONT_MATTER.exec(markdown.replace(/\r\n/g, '\n'))?.[1]
+ return front === undefined ? undefined : CANONICAL_FIELD.exec(front)?.[1]
+}
+
+function twinPath(
+ href: string,
+ base: string,
+ origin: string
+): { path: string } | { problem: string } {
+ try {
+ const url = new URL(href, base)
+ if (url.origin !== origin)
+ return { problem: `advertises ${href}, which is not on ${origin}` }
+ return { path: decodeURI(url.pathname) }
+ } catch {
+ return { problem: `advertises ${href}, which is not a valid URL` }
+ }
+}
+
+function pageProblem(
+ { route, html }: BuiltPage,
+ readTwin: (path: string) => string | undefined,
+ origin: string
+): { advertises: boolean; canonicalChecked?: boolean; problem?: string } {
+ const links = linkElements(parse(html))
+ const alternate = links.find(isMarkdownAlternate)
+ if (alternate === undefined) return { advertises: false }
+ const href = alternate.get('href')
+ if (!href) {
+ return {
+ advertises: true,
+ problem: `${route}: has a text/markdown link with no readable href`
+ }
+ }
+ const resolved = twinPath(href, `${origin}${route}`, origin)
+ if ('problem' in resolved)
+ return { advertises: true, problem: `${route}: ${resolved.problem}` }
+ const { path } = resolved
+ const twin = readTwin(path)
+ if (twin === undefined) {
+ return {
+ advertises: true,
+ problem: `${route}: advertises ${path}, which was not built`
+ }
+ }
+ const actual = twinCanonical(twin)
+ if (actual === undefined) {
+ return {
+ advertises: true,
+ problem: `${route}: ${path} has no canonical in its front matter`
+ }
+ }
+ const expected = links.find((link) => hasRel(link, 'canonical'))?.get('href')
+ if (expected === undefined) return { advertises: true }
+ return {
+ advertises: true,
+ canonicalChecked: true,
+ problem:
+ actual === expected
+ ? undefined
+ : `${route}: ${path} names canonical ${actual}, the page says ${expected}`
+ }
+}
+
+/**
+ * One problem per page whose advertised markdown copy has an unreadable href,
+ * is missing, has no `canonical:` front matter, or names a different page than
+ * the HTML does. A page without `rel="canonical"` (every page of a
+ * non-indexable build) has nothing to compare against, so only its copy's
+ * existence is checked. `readTwin` returns the twin's text, or undefined when
+ * no such file was built.
+ */
+export function auditMarkdownAlternates(
+ pages: readonly BuiltPage[],
+ readTwin: (path: string) => string | undefined,
+ origin: string
+): MarkdownAlternateReport {
+ const results = pages.map((page) => pageProblem(page, readTwin, origin))
+ return {
+ advertised: results.filter((result) => result.advertises).length,
+ canonicalChecked: results.filter((result) => result.canonicalChecked)
+ .length,
+ problems: results.flatMap((result) =>
+ result.problem === undefined ? [] : [result.problem]
+ )
+ }
+}
+
+/** Why too few built pages advertise a markdown copy, or undefined. */
+export function advertisementShortfall(
+ advertised: number,
+ builtPages: number
+): string | undefined {
+ const required = Math.max(1, Math.ceil(builtPages * MIN_ADVERTISED_SHARE))
+ if (advertised >= required) return undefined
+ return (
+ `only ${advertised} of ${builtPages} built pages advertise a markdown copy; ` +
+ `at least ${required} (${MIN_ADVERTISED_SHARE * 100}%) must. Pages lost ` +
+ 'their , or the build is incomplete.'
+ )
+}