diff --git a/apps/website/src/config/indexing.test.ts b/apps/website/src/config/indexing.test.ts index 48f996a4dc1..9eaf9157ad8 100644 --- a/apps/website/src/config/indexing.test.ts +++ b/apps/website/src/config/indexing.test.ts @@ -1,5 +1,15 @@ +import { readFileSync, readdirSync } from 'node:fs' +import { dirname, join, relative, sep } from 'node:path' +import { fileURLToPath } from 'node:url' import { describe, expect, it, vi } from 'vitest' -import { isExcludedFromSitemap, isNoindexPathname } from './indexing' + +import { modelsBuildRoutes } from '../integrations/workshop-release-gate' +import { routeOf } from '../utils/hreflangRoutes' +import { + NOINDEX_ROUTES, + isExcludedFromSitemap, + isNoindexPathname +} from './indexing' describe('indexing policy', () => { it('excludes render pages while keeping the public Models marketing routes', () => { @@ -58,7 +68,8 @@ describe('indexing policy', () => { '/zh-CN/checkout-opening/', '/checkout-return', '/zh-CN/checkout-return/', - '/zh-CN/login' + '/zh-CN/login', + '/platform/serverless-animation/' ])('marks %s as noindex', (pathname) => { expect(isNoindexPathname(pathname)).toBe(true) expect(isExcludedFromSitemap(`https://comfy.org${pathname}`)).toBe(true) @@ -89,3 +100,66 @@ describe('indexing policy', () => { ).toBe(true) }) }) + +const pagesDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'pages') + +const astroFiles = (dir: string): string[] => + readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const full = join(dir, entry.name) + if (entry.isDirectory()) return astroFiles(full) + return entry.name.endsWith('.astro') ? [full] : [] + }) + +const pageRoute = (file: string) => + routeOf(`/src/pages/${relative(pagesDir, file).split(sep).join('/')}`) + +describe('pages that are noindex on their own', () => { + const isNoindexOnItsOwn = (file: string) => { + const source = readFileSync(file, 'utf8') + return ( + /\bAuthLayout\b/.test(source) || + /\snoindex(?=[\s/>=])/.test(source.replace(/^---[\s\S]*?\n---/, '')) + ) + } + + const routes = astroFiles(pagesDir) + .filter(isNoindexOnItsOwn) + .map(pageRoute) + .filter((route) => route !== '/404/') + + it('finds the pages that pass the prop or use AuthLayout', () => { + expect(routes).toContain('/privacy-policy/') + expect(routes).toContain('/comfy-agent/') + expect(routes).toContain('/login/') + }) + + it.for(routes)('lists %s in NOINDEX_ROUTES', (route) => { + expect(isNoindexPathname(route)).toBe(true) + }) +}) + +describe('NOINDEX_ROUTES', () => { + const toMatcher = (route: string) => + new RegExp( + `^${route + .replace(/\/$/, '') + .split('/') + .map((segment) => + segment.startsWith('[...') + ? '.+' + : segment.startsWith('[') + ? '[^/]+' + : segment.replace(/[.*+?^${}()|\\]/g, '\\$&') + ) + .join('/')}$` + ) + + const builtRoutes = [ + ...astroFiles(pagesDir).map(pageRoute), + ...modelsBuildRoutes(true).map(({ pattern }) => pattern) + ].map(toMatcher) + + it.for(NOINDEX_ROUTES)('%s is a route this site builds', (route) => { + expect(builtRoutes.some((matcher) => matcher.test(route))).toBe(true) + }) +}) diff --git a/apps/website/src/config/indexing.ts b/apps/website/src/config/indexing.ts index 0f0bca2059b..1d996969f56 100644 --- a/apps/website/src/config/indexing.ts +++ b/apps/website/src/config/indexing.ts @@ -7,7 +7,7 @@ const PLACEHOLDER_PATHNAMES = ['/case-studies', '/videos', '/demos'] as const const ALL_LOCALE_PREFIXES = LOCALE_CODES.map((locale) => LOCALES[locale].prefix) -const NOINDEX_ROUTES = [ +export const NOINDEX_ROUTES = [ ...PAYMENT_STATUSES.map((status) => `/payment/${status}`), '/individual-submission', '/booking-confirmation', @@ -22,8 +22,9 @@ const NOINDEX_ROUTES = [ '/checkout-return', '/privacy-policy', '/terms-of-service', + '/platform/serverless-animation', ...PLACEHOLDER_PATHNAMES -] +] as const const NOINDEX_PATHNAMES = new Set([ ...ALL_LOCALE_PREFIXES.flatMap((prefix) => diff --git a/apps/website/src/config/llms-txt.test.ts b/apps/website/src/config/llms-txt.test.ts index 31d63c71067..d8848a6e73b 100644 --- a/apps/website/src/config/llms-txt.test.ts +++ b/apps/website/src/config/llms-txt.test.ts @@ -42,7 +42,6 @@ const EXCLUDED_PAGES = new Set([ '/case-studies', // "Coming Soon" placeholder '/videos', // "Coming Soon" placeholder '/demos', // index is a "Coming Soon" placeholder; the demo pages are listed - '/platform/serverless-animation', // noindex temporary motion study, not a real page '/workshop', // build-gated; static public/llms.txt cannot vary by build shape '/video-sitemap.xml' // machine-readable sitemap output, not a page for agents to read ]) diff --git a/apps/website/src/pages/platform/serverless-animation.astro b/apps/website/src/pages/platform/serverless-animation.astro index 00911216089..d6aa59056c6 100644 --- a/apps/website/src/pages/platform/serverless-animation.astro +++ b/apps/website/src/pages/platform/serverless-animation.astro @@ -8,7 +8,6 @@ import ServerlessWorkerAnimation from '../../templates/platform/ServerlessWorker