From 1aea3ed32491cc2931ba0bdaf7ea1efb70a09d92 Mon Sep 17 00:00:00 2001 From: Maanil Verma Date: Wed, 30 Sep 2026 02:41:03 +0530 Subject: [PATCH 1/5] fix(website): let NOINDEX_ROUTES decide noindex for /platform/serverless-animation --- apps/website/src/config/indexing.test.ts | 3 ++- apps/website/src/config/indexing.ts | 1 + apps/website/src/pages/platform/serverless-animation.astro | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/website/src/config/indexing.test.ts b/apps/website/src/config/indexing.test.ts index 48f996a4dc1..8ab274d6a7d 100644 --- a/apps/website/src/config/indexing.test.ts +++ b/apps/website/src/config/indexing.test.ts @@ -58,7 +58,8 @@ describe('indexing policy', () => { '/zh-CN/checkout-opening/', '/checkout-return', '/zh-CN/checkout-return/', - '/zh-CN/login' + '/zh-CN/login', + '/platform/serverless-animation/' ])('marks %s as noindex', (pathname) => { expect(isNoindexPathname(pathname)).toBe(true) expect(isExcludedFromSitemap(`https://comfy.org${pathname}`)).toBe(true) diff --git a/apps/website/src/config/indexing.ts b/apps/website/src/config/indexing.ts index 0f0bca2059b..5042ffae3ee 100644 --- a/apps/website/src/config/indexing.ts +++ b/apps/website/src/config/indexing.ts @@ -22,6 +22,7 @@ const NOINDEX_ROUTES = [ '/checkout-return', '/privacy-policy', '/terms-of-service', + '/platform/serverless-animation', ...PLACEHOLDER_PATHNAMES ] diff --git a/apps/website/src/pages/platform/serverless-animation.astro b/apps/website/src/pages/platform/serverless-animation.astro index 00911216089..d6aa59056c6 100644 --- a/apps/website/src/pages/platform/serverless-animation.astro +++ b/apps/website/src/pages/platform/serverless-animation.astro @@ -8,7 +8,6 @@ import ServerlessWorkerAnimation from '../../templates/platform/ServerlessWorker
From db3bbcd3060c0c74fd1060936c5cbca15e31e1fe Mon Sep 17 00:00:00 2001 From: Maanil Verma Date: Wed, 30 Sep 2026 03:19:25 +0530 Subject: [PATCH 2/5] test(website): drop the llms.txt exclusion that NOINDEX_ROUTES now covers --- apps/website/src/config/llms-txt.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/website/src/config/llms-txt.test.ts b/apps/website/src/config/llms-txt.test.ts index 31d63c71067..d8848a6e73b 100644 --- a/apps/website/src/config/llms-txt.test.ts +++ b/apps/website/src/config/llms-txt.test.ts @@ -42,7 +42,6 @@ const EXCLUDED_PAGES = new Set([ '/case-studies', // "Coming Soon" placeholder '/videos', // "Coming Soon" placeholder '/demos', // index is a "Coming Soon" placeholder; the demo pages are listed - '/platform/serverless-animation', // noindex temporary motion study, not a real page '/workshop', // build-gated; static public/llms.txt cannot vary by build shape '/video-sitemap.xml' // machine-readable sitemap output, not a page for agents to read ]) From 6e70fde19fe54b37028850bb71f76f193692aacf Mon Sep 17 00:00:00 2001 From: Maanil Verma Date: Wed, 30 Sep 2026 03:20:19 +0530 Subject: [PATCH 3/5] test(website): fail when a page passes noindex but is missing from NOINDEX_ROUTES --- apps/website/src/config/indexing.test.ts | 37 ++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/apps/website/src/config/indexing.test.ts b/apps/website/src/config/indexing.test.ts index 8ab274d6a7d..8dc742dbf62 100644 --- a/apps/website/src/config/indexing.test.ts +++ b/apps/website/src/config/indexing.test.ts @@ -1,4 +1,9 @@ +import { readFileSync, readdirSync } from 'node:fs' +import { dirname, join, relative, sep } from 'node:path' +import { fileURLToPath } from 'node:url' import { describe, expect, it, vi } from 'vitest' + +import { routeOf } from '../utils/hreflangRoutes' import { isExcludedFromSitemap, isNoindexPathname } from './indexing' describe('indexing policy', () => { @@ -90,3 +95,35 @@ describe('indexing policy', () => { ).toBe(true) }) }) + +describe('pages that pass the noindex prop', () => { + const pagesDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'pages') + + const astroFiles = (dir: string): string[] => + readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const full = join(dir, entry.name) + if (entry.isDirectory()) return astroFiles(full) + return entry.name.endsWith('.astro') ? [full] : [] + }) + + const passesNoindex = (file: string) => + /\snoindex(?=[\s/>=])/.test( + readFileSync(file, 'utf8').replace(/^---[\s\S]*?\n---/, '') + ) + + const routes = astroFiles(pagesDir) + .filter(passesNoindex) + .map((file) => + routeOf(`/src/pages/${relative(pagesDir, file).split(sep).join('/')}`) + ) + .filter((route) => route !== '/404/') + + it('finds the pages that set the prop', () => { + expect(routes).toContain('/privacy-policy/') + expect(routes).toContain('/comfy-agent/') + }) + + it.for(routes)('lists %s in NOINDEX_ROUTES', (route) => { + expect(isNoindexPathname(route)).toBe(true) + }) +}) From 96994c2ed063ecf7803ee2cdc77bc79b0ad58d83 Mon Sep 17 00:00:00 2001 From: Maanil Verma Date: Wed, 30 Sep 2026 03:36:26 +0530 Subject: [PATCH 4/5] test(website): treat AuthLayout as noindex and fail on a stale NOINDEX_ROUTES entry --- apps/website/src/config/indexing.test.ts | 70 ++++++++++++++++++------ apps/website/src/config/indexing.ts | 2 +- 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/apps/website/src/config/indexing.test.ts b/apps/website/src/config/indexing.test.ts index 8dc742dbf62..9eaf9157ad8 100644 --- a/apps/website/src/config/indexing.test.ts +++ b/apps/website/src/config/indexing.test.ts @@ -3,8 +3,13 @@ import { dirname, join, relative, sep } from 'node:path' import { fileURLToPath } from 'node:url' import { describe, expect, it, vi } from 'vitest' +import { modelsBuildRoutes } from '../integrations/workshop-release-gate' import { routeOf } from '../utils/hreflangRoutes' -import { isExcludedFromSitemap, isNoindexPathname } from './indexing' +import { + NOINDEX_ROUTES, + isExcludedFromSitemap, + isNoindexPathname +} from './indexing' describe('indexing policy', () => { it('excludes render pages while keeping the public Models marketing routes', () => { @@ -96,34 +101,65 @@ describe('indexing policy', () => { }) }) -describe('pages that pass the noindex prop', () => { - const pagesDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'pages') +const pagesDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'pages') - const astroFiles = (dir: string): string[] => - readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { - const full = join(dir, entry.name) - if (entry.isDirectory()) return astroFiles(full) - return entry.name.endsWith('.astro') ? [full] : [] - }) +const astroFiles = (dir: string): string[] => + readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const full = join(dir, entry.name) + if (entry.isDirectory()) return astroFiles(full) + return entry.name.endsWith('.astro') ? [full] : [] + }) + +const pageRoute = (file: string) => + routeOf(`/src/pages/${relative(pagesDir, file).split(sep).join('/')}`) - const passesNoindex = (file: string) => - /\snoindex(?=[\s/>=])/.test( - readFileSync(file, 'utf8').replace(/^---[\s\S]*?\n---/, '') +describe('pages that are noindex on their own', () => { + const isNoindexOnItsOwn = (file: string) => { + const source = readFileSync(file, 'utf8') + return ( + /\bAuthLayout\b/.test(source) || + /\snoindex(?=[\s/>=])/.test(source.replace(/^---[\s\S]*?\n---/, '')) ) + } const routes = astroFiles(pagesDir) - .filter(passesNoindex) - .map((file) => - routeOf(`/src/pages/${relative(pagesDir, file).split(sep).join('/')}`) - ) + .filter(isNoindexOnItsOwn) + .map(pageRoute) .filter((route) => route !== '/404/') - it('finds the pages that set the prop', () => { + it('finds the pages that pass the prop or use AuthLayout', () => { expect(routes).toContain('/privacy-policy/') expect(routes).toContain('/comfy-agent/') + expect(routes).toContain('/login/') }) it.for(routes)('lists %s in NOINDEX_ROUTES', (route) => { expect(isNoindexPathname(route)).toBe(true) }) }) + +describe('NOINDEX_ROUTES', () => { + const toMatcher = (route: string) => + new RegExp( + `^${route + .replace(/\/$/, '') + .split('/') + .map((segment) => + segment.startsWith('[...') + ? '.+' + : segment.startsWith('[') + ? '[^/]+' + : segment.replace(/[.*+?^${}()|\\]/g, '\\$&') + ) + .join('/')}$` + ) + + const builtRoutes = [ + ...astroFiles(pagesDir).map(pageRoute), + ...modelsBuildRoutes(true).map(({ pattern }) => pattern) + ].map(toMatcher) + + it.for(NOINDEX_ROUTES)('%s is a route this site builds', (route) => { + expect(builtRoutes.some((matcher) => matcher.test(route))).toBe(true) + }) +}) diff --git a/apps/website/src/config/indexing.ts b/apps/website/src/config/indexing.ts index 5042ffae3ee..9a56856eba3 100644 --- a/apps/website/src/config/indexing.ts +++ b/apps/website/src/config/indexing.ts @@ -7,7 +7,7 @@ const PLACEHOLDER_PATHNAMES = ['/case-studies', '/videos', '/demos'] as const const ALL_LOCALE_PREFIXES = LOCALE_CODES.map((locale) => LOCALES[locale].prefix) -const NOINDEX_ROUTES = [ +export const NOINDEX_ROUTES = [ ...PAYMENT_STATUSES.map((status) => `/payment/${status}`), '/individual-submission', '/booking-confirmation', From 1241ba8281e48e443aeebf721ad7b8b31a904a22 Mon Sep 17 00:00:00 2001 From: Maanil Verma Date: Wed, 30 Sep 2026 04:15:23 +0530 Subject: [PATCH 5/5] refactor(website): make NOINDEX_ROUTES readonly with as const --- apps/website/src/config/indexing.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/website/src/config/indexing.ts b/apps/website/src/config/indexing.ts index 9a56856eba3..1d996969f56 100644 --- a/apps/website/src/config/indexing.ts +++ b/apps/website/src/config/indexing.ts @@ -24,7 +24,7 @@ export const NOINDEX_ROUTES = [ '/terms-of-service', '/platform/serverless-animation', ...PLACEHOLDER_PATHNAMES -] +] as const const NOINDEX_PATHNAMES = new Set([ ...ALL_LOCALE_PREFIXES.flatMap((prefix) =>