Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 76 additions & 2 deletions apps/website/src/config/indexing.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,15 @@
import { readFileSync, readdirSync } from 'node:fs'
import { dirname, join, relative, sep } from 'node:path'
import { fileURLToPath } from 'node:url'
import { describe, expect, it, vi } from 'vitest'
import { isExcludedFromSitemap, isNoindexPathname } from './indexing'

import { modelsBuildRoutes } from '../integrations/workshop-release-gate'
import { routeOf } from '../utils/hreflangRoutes'
import {
NOINDEX_ROUTES,
isExcludedFromSitemap,
isNoindexPathname
} from './indexing'

describe('indexing policy', () => {
it('excludes render pages while keeping the public Models marketing routes', () => {
Expand Down Expand Up @@ -58,7 +68,8 @@ describe('indexing policy', () => {
'/zh-CN/checkout-opening/',
'/checkout-return',
'/zh-CN/checkout-return/',
'/zh-CN/login'
'/zh-CN/login',
'/platform/serverless-animation/'
Comment thread
benjcooley marked this conversation as resolved.
Resolved
])('marks %s as noindex', (pathname) => {
expect(isNoindexPathname(pathname)).toBe(true)
expect(isExcludedFromSitemap(`https://comfy.org${pathname}`)).toBe(true)
Expand Down Expand Up @@ -89,3 +100,66 @@ describe('indexing policy', () => {
).toBe(true)
})
})

const pagesDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'pages')

const astroFiles = (dir: string): string[] =>
readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
const full = join(dir, entry.name)
if (entry.isDirectory()) return astroFiles(full)
return entry.name.endsWith('.astro') ? [full] : []
})

const pageRoute = (file: string) =>
routeOf(`/src/pages/${relative(pagesDir, file).split(sep).join('/')}`)

describe('pages that are noindex on their own', () => {
const isNoindexOnItsOwn = (file: string) => {
const source = readFileSync(file, 'utf8')
return (
/\bAuthLayout\b/.test(source) ||
/\snoindex(?=[\s/>=])/.test(source.replace(/^---[\s\S]*?\n---/, ''))
)
Comment thread
benjcooley marked this conversation as resolved.
Resolved
}

const routes = astroFiles(pagesDir)
.filter(isNoindexOnItsOwn)
.map(pageRoute)
.filter((route) => route !== '/404/')

it('finds the pages that pass the prop or use AuthLayout', () => {
expect(routes).toContain('/privacy-policy/')
expect(routes).toContain('/comfy-agent/')
expect(routes).toContain('/login/')
})

it.for(routes)('lists %s in NOINDEX_ROUTES', (route) => {
expect(isNoindexPathname(route)).toBe(true)
})
})

describe('NOINDEX_ROUTES', () => {
const toMatcher = (route: string) =>
new RegExp(
`^${route
.replace(/\/$/, '')
.split('/')
.map((segment) =>
segment.startsWith('[...')
? '.+'
: segment.startsWith('[')
? '[^/]+'
: segment.replace(/[.*+?^${}()|\\]/g, '\\$&')
)
.join('/')}$`
)

const builtRoutes = [
...astroFiles(pagesDir).map(pageRoute),
...modelsBuildRoutes(true).map(({ pattern }) => pattern)
].map(toMatcher)

it.for(NOINDEX_ROUTES)('%s is a route this site builds', (route) => {
expect(builtRoutes.some((matcher) => matcher.test(route))).toBe(true)
})
})
5 changes: 3 additions & 2 deletions apps/website/src/config/indexing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ const PLACEHOLDER_PATHNAMES = ['/case-studies', '/videos', '/demos'] as const

const ALL_LOCALE_PREFIXES = LOCALE_CODES.map((locale) => LOCALES[locale].prefix)

const NOINDEX_ROUTES = [
export const NOINDEX_ROUTES = [
Comment thread
benjcooley marked this conversation as resolved.
Resolved
...PAYMENT_STATUSES.map((status) => `/payment/${status}`),
'/individual-submission',
'/booking-confirmation',
Expand All @@ -22,8 +22,9 @@ const NOINDEX_ROUTES = [
'/checkout-return',
'/privacy-policy',
'/terms-of-service',
'/platform/serverless-animation',
Comment thread
benjcooley marked this conversation as resolved.
Resolved
...PLACEHOLDER_PATHNAMES
]
] as const

const NOINDEX_PATHNAMES = new Set([
...ALL_LOCALE_PREFIXES.flatMap((prefix) =>
Expand Down
1 change: 0 additions & 1 deletion apps/website/src/config/llms-txt.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ const EXCLUDED_PAGES = new Set([
'/case-studies', // "Coming Soon" placeholder
'/videos', // "Coming Soon" placeholder
'/demos', // index is a "Coming Soon" placeholder; the demo pages are listed
'/platform/serverless-animation', // noindex temporary motion study, not a real page
'/workshop', // build-gated; static public/llms.txt cannot vary by build shape
'/video-sitemap.xml' // machine-readable sitemap output, not a page for agents to read
])
Expand Down
1 change: 0 additions & 1 deletion apps/website/src/pages/platform/serverless-animation.astro
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import ServerlessWorkerAnimation from '../../templates/platform/ServerlessWorker
<BaseLayout
title="Serverless animation comparison · Comfy"
description="Temporary side-by-side serverless animation study."
noindex
>
Comment thread
MaanilVerma marked this conversation as resolved.
Resolved
<section class="mx-auto w-full max-w-[1600px] px-6 py-16 sm:px-10 lg:px-12">
<header class="mb-10 max-w-3xl">
Expand Down
Loading