From 74415ce770590a0e297c4df2281ce276cc8e1875 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 05:17:31 +0000 Subject: [PATCH 01/35] test(website): enforce router drift checks in CI Fixes #19709 --- .github/workflows/ci-website-unit.yaml | 2 ++ .../src/config/router-providers.test.ts | 33 ++++++++++++++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-website-unit.yaml b/.github/workflows/ci-website-unit.yaml index cae0ff2eeef..66b7ea8aa65 100644 --- a/.github/workflows/ci-website-unit.yaml +++ b/.github/workflows/ci-website-unit.yaml @@ -37,6 +37,8 @@ jobs: uses: ./.github/actions/setup-frontend - name: Run website unit tests with coverage + env: + CI: 1 run: pnpm --filter @comfyorg/website test:coverage - name: Upload website coverage to Codecov diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index 8b4408c854b..3c21712dd41 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -1,7 +1,7 @@ // @vitest-environment node import { z } from 'astro/zod' -import { describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { ROUTER_COMFY_ONLY_PREVIEW, @@ -20,8 +20,8 @@ const ROUTER_SCHEMAS = // The API spec and the docs are the source of truth and live outside this // repository, so the drift checks reach the network. They run in CI and skip -// elsewhere, or when a source cannot be fetched, so an outage never fails a -// build: the day the spec changes, CI is where it shows. +// elsewhere. CI treats an unavailable source as a failure so missing coverage +// cannot produce a green build. const skipReason = process.env.CI ? null : 'checks the published API spec; runs in CI only (set CI=1 to run it here)' @@ -30,7 +30,9 @@ async function fetchDocs(url: string): Promise { let response: Response try { response = await fetch(url, { signal: AbortSignal.timeout(15_000) }) - } catch { + } catch (error) { + if (process.env.CI) + throw new Error(`Could not fetch ${url}`, { cause: error }) return null } if (!response.ok) throw new Error(`${url} responded ${response.status}`) @@ -77,6 +79,29 @@ function parseCoverageTable(markdown: string): { return { providers, rows } } +describe('Router provider source availability', () => { + afterEach(() => { + vi.unstubAllEnvs() + vi.unstubAllGlobals() + }) + + it('skips an unavailable source outside CI', async () => { + vi.stubEnv('CI', '') + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline'))) + + await expect(fetchDocs(PROVIDERS_PAGE)).resolves.toBeNull() + }) + + it('fails when a required source is unavailable in CI', async () => { + vi.stubEnv('CI', '1') + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline'))) + + await expect(fetchDocs(PROVIDERS_PAGE)).rejects.toThrow( + `Could not fetch ${PROVIDERS_PAGE}` + ) + }) +}) + describe('Router provider coverage', () => { it('lists each model once, alphabetically', () => { const names = ROUTER_PROVIDER_COVERAGE.map((row) => row.name) From e3586c18e9b6df3d1dd03c30f9c4db9c834a1374 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 05:21:18 +0000 Subject: [PATCH 02/35] test(website): rely on shared Vitest cleanup --- apps/website/src/config/router-providers.test.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index 3c21712dd41..faf27a80347 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -1,7 +1,7 @@ // @vitest-environment node import { z } from 'astro/zod' -import { afterEach, describe, expect, it, vi } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { ROUTER_COMFY_ONLY_PREVIEW, @@ -80,11 +80,6 @@ function parseCoverageTable(markdown: string): { } describe('Router provider source availability', () => { - afterEach(() => { - vi.unstubAllEnvs() - vi.unstubAllGlobals() - }) - it('skips an unavailable source outside CI', async () => { vi.stubEnv('CI', '') vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline'))) From baeb94526866efd0e955f9d2fb5f660dbf655f59 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 08:39:34 +0000 Subject: [PATCH 03/35] ci(website): rely on runner CI environment Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4153017470 --- .github/workflows/ci-website-unit.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-website-unit.yaml b/.github/workflows/ci-website-unit.yaml index 66b7ea8aa65..cae0ff2eeef 100644 --- a/.github/workflows/ci-website-unit.yaml +++ b/.github/workflows/ci-website-unit.yaml @@ -37,8 +37,6 @@ jobs: uses: ./.github/actions/setup-frontend - name: Run website unit tests with coverage - env: - CI: 1 run: pnpm --filter @comfyorg/website test:coverage - name: Upload website coverage to Codecov From 0c09930d20f656ec4794500e00c56941be55e9bd Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 08:40:01 +0000 Subject: [PATCH 04/35] test(website): parse the CI flag explicitly Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4152145517 --- apps/website/src/config/router-providers.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index faf27a80347..899ce797f90 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -17,12 +17,14 @@ const MODELS_PAGE = `${ROUTER_DOCS}/models.md` // The per-model API spec, as the backend publishes it for the docs site. const ROUTER_SCHEMAS = 'https://raw.githubusercontent.com/Comfy-Org/docs/main/router-schemas' +const isCI = () => + ['1', 'true'].includes(process.env.CI?.toLowerCase() ?? '') // The API spec and the docs are the source of truth and live outside this // repository, so the drift checks reach the network. They run in CI and skip // elsewhere. CI treats an unavailable source as a failure so missing coverage // cannot produce a green build. -const skipReason = process.env.CI +const skipReason = isCI() ? null : 'checks the published API spec; runs in CI only (set CI=1 to run it here)' @@ -31,7 +33,7 @@ async function fetchDocs(url: string): Promise { try { response = await fetch(url, { signal: AbortSignal.timeout(15_000) }) } catch (error) { - if (process.env.CI) + if (isCI()) throw new Error(`Could not fetch ${url}`, { cause: error }) return null } From f90412f6650f90dd589bac54034e1ccb31f7a50e Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 08:40:55 +0000 Subject: [PATCH 05/35] test(website): normalize source fetch failures Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4152145514 --- apps/website/src/config/router-providers.test.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index 899ce797f90..6ca83b80037 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -29,16 +29,15 @@ const skipReason = isCI() : 'checks the published API spec; runs in CI only (set CI=1 to run it here)' async function fetchDocs(url: string): Promise { - let response: Response try { - response = await fetch(url, { signal: AbortSignal.timeout(15_000) }) + const response = await fetch(url, { signal: AbortSignal.timeout(15_000) }) + if (!response.ok) throw new Error(`${url} responded ${response.status}`) + return await response.text() } catch (error) { if (isCI()) throw new Error(`Could not fetch ${url}`, { cause: error }) return null } - if (!response.ok) throw new Error(`${url} responded ${response.status}`) - return response.text() } interface DocsCoverageRow { From 6338c203be246b56360bc2913b16c02b96dd7faa Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 08:41:16 +0000 Subject: [PATCH 06/35] test(website): retry published router sources Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4152145506 --- .../src/config/router-providers.test.ts | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index 6ca83b80037..c65cf71dc4d 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -17,6 +17,8 @@ const MODELS_PAGE = `${ROUTER_DOCS}/models.md` // The per-model API spec, as the backend publishes it for the docs site. const ROUTER_SCHEMAS = 'https://raw.githubusercontent.com/Comfy-Org/docs/main/router-schemas' +const FETCH_ATTEMPTS = 3 +const FETCH_TIMEOUT_MS = 4_000 const isCI = () => ['1', 'true'].includes(process.env.CI?.toLowerCase() ?? '') @@ -29,15 +31,23 @@ const skipReason = isCI() : 'checks the published API spec; runs in CI only (set CI=1 to run it here)' async function fetchDocs(url: string): Promise { - try { - const response = await fetch(url, { signal: AbortSignal.timeout(15_000) }) - if (!response.ok) throw new Error(`${url} responded ${response.status}`) - return await response.text() - } catch (error) { - if (isCI()) - throw new Error(`Could not fetch ${url}`, { cause: error }) - return null + const attempts = isCI() ? FETCH_ATTEMPTS : 1 + let lastError: unknown + for (let attempt = 0; attempt < attempts; attempt++) { + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) + }) + if (!response.ok) throw new Error(`${url} responded ${response.status}`) + return await response.text() + } catch (error) { + lastError = error + if (attempt + 1 < attempts) + await new Promise((resolve) => setTimeout(resolve, 250 * 2 ** attempt)) + } } + if (isCI()) throw new Error(`Could not fetch ${url}`, { cause: lastError }) + return null } interface DocsCoverageRow { From 3b2c0556eb236ca34ef0ce8bd62f952a2bd252a1 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 08:42:54 +0000 Subject: [PATCH 07/35] test(website): budget time for source retries Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4152145512 --- apps/website/src/config/router-providers.test.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index c65cf71dc4d..2cd8d3157b4 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -19,8 +19,7 @@ const ROUTER_SCHEMAS = 'https://raw.githubusercontent.com/Comfy-Org/docs/main/router-schemas' const FETCH_ATTEMPTS = 3 const FETCH_TIMEOUT_MS = 4_000 -const isCI = () => - ['1', 'true'].includes(process.env.CI?.toLowerCase() ?? '') +const isCI = () => ['1', 'true'].includes(process.env.CI?.toLowerCase() ?? '') // The API spec and the docs are the source of truth and live outside this // repository, so the drift checks reach the network. They run in CI and skip @@ -138,7 +137,7 @@ describe('Router provider coverage', () => { .sort() ) ).toEqual(ROUTER_PROVIDER_COVERAGE.map((row) => [...row.providers].sort())) - }) + }, 15_000) it('lists every model the docs show with an alternate provider', async (ctx) => { if (skipReason) return ctx.skip(skipReason) @@ -162,7 +161,7 @@ describe('Router provider coverage', () => { name: row.docsName ?? row.name })).sort(byDocsUrl) ) - }) + }, 15_000) it('previews catalog models that only Comfy serves', async (ctx) => { if (skipReason) return ctx.skip(skipReason) @@ -178,5 +177,5 @@ describe('Router provider coverage', () => { expect(catalog).toContain(`[${name}](${path})`) expect(coverage).not.toContain(path) } - }) + }, 15_000) }) From d45169bbaa8ced7c6d753f0c07a33344edecb7a1 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 08:43:13 +0000 Subject: [PATCH 08/35] test(website): remove unreachable source skips Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4152145516 --- .../src/config/router-providers.test.ts | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index 2cd8d3157b4..25ec3ef5f20 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -29,8 +29,8 @@ const skipReason = isCI() ? null : 'checks the published API spec; runs in CI only (set CI=1 to run it here)' -async function fetchDocs(url: string): Promise { - const attempts = isCI() ? FETCH_ATTEMPTS : 1 +async function fetchDocs(url: string): Promise { + const attempts = FETCH_ATTEMPTS let lastError: unknown for (let attempt = 0; attempt < attempts; attempt++) { try { @@ -45,8 +45,7 @@ async function fetchDocs(url: string): Promise { await new Promise((resolve) => setTimeout(resolve, 250 * 2 ** attempt)) } } - if (isCI()) throw new Error(`Could not fetch ${url}`, { cause: lastError }) - return null + throw new Error(`Could not fetch ${url}`, { cause: lastError }) } interface DocsCoverageRow { @@ -90,13 +89,6 @@ function parseCoverageTable(markdown: string): { } describe('Router provider source availability', () => { - it('skips an unavailable source outside CI', async () => { - vi.stubEnv('CI', '') - vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline'))) - - await expect(fetchDocs(PROVIDERS_PAGE)).resolves.toBeNull() - }) - it('fails when a required source is unavailable in CI', async () => { vi.stubEnv('CI', '1') vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline'))) @@ -122,7 +114,6 @@ describe('Router provider coverage', () => { fetchDocs(`${ROUTER_SCHEMAS}/${row.modelId}.json`) ) ) - if (specs.includes(null)) return ctx.skip('could not fetch the API spec') const altProviders = z.object({ 'x-comfy-router-alt-providers': z @@ -132,7 +123,7 @@ describe('Router provider coverage', () => { expect( specs.map((spec) => altProviders - .parse(JSON.parse(spec ?? '{}')) + .parse(JSON.parse(spec)) ['x-comfy-router-alt-providers'].map(({ provider }) => provider) .sort() ) @@ -142,7 +133,6 @@ describe('Router provider coverage', () => { it('lists every model the docs show with an alternate provider', async (ctx) => { if (skipReason) return ctx.skip(skipReason) const markdown = await fetchDocs(PROVIDERS_PAGE) - if (markdown === null) return ctx.skip(`could not fetch ${PROVIDERS_PAGE}`) const docs = parseCoverageTable(markdown) expect(docs.providers).toEqual( @@ -169,8 +159,6 @@ describe('Router provider coverage', () => { fetchDocs(MODELS_PAGE), fetchDocs(PROVIDERS_PAGE) ]) - if (catalog === null || coverage === null) - return ctx.skip('could not fetch the docs catalog') for (const { name, docsUrl } of ROUTER_COMFY_ONLY_PREVIEW) { const path = docsUrl.slice(DOCS_ORIGIN.length) From 4e516b307f176ffe8377ea2c79e8e8d8f0c792ec Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 08:43:38 +0000 Subject: [PATCH 09/35] test(website): cover deterministic provider drift Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4152157011 --- .../src/config/router-providers.test.ts | 49 ++++++++++++++----- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index 25ec3ef5f20..fbece46b6a6 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -55,6 +55,22 @@ interface DocsCoverageRow { providers: string[] } +const altProvidersSchema = z.object({ + 'x-comfy-router-alt-providers': z + .array(z.object({ provider: z.string() })) + .default([]) +}) + +function expectAlternateProviders(specs: string[], expected: string[][]): void { + const actual = specs.map((spec) => + altProvidersSchema + .parse(JSON.parse(spec)) + ['x-comfy-router-alt-providers'].map(({ provider }) => provider) + .sort() + ) + expect(actual).toEqual(expected.map((providers) => [...providers].sort())) +} + /** Reads the "Provider coverage" table from the docs page's Markdown. */ function parseCoverageTable(markdown: string): { providers: string[] @@ -97,6 +113,22 @@ describe('Router provider source availability', () => { `Could not fetch ${PROVIDERS_PAGE}` ) }) + + it('detects provider drift from a successful source response', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + 'x-comfy-router-alt-providers': [{ provider: 'anthropic' }] + }) + ) + ) + ) + + const spec = await fetchDocs(PROVIDERS_PAGE) + expect(() => expectAlternateProviders([spec], [['openai']])).toThrow() + }) }) describe('Router provider coverage', () => { @@ -115,19 +147,10 @@ describe('Router provider coverage', () => { ) ) - const altProviders = z.object({ - 'x-comfy-router-alt-providers': z - .array(z.object({ provider: z.string() })) - .default([]) - }) - expect( - specs.map((spec) => - altProviders - .parse(JSON.parse(spec)) - ['x-comfy-router-alt-providers'].map(({ provider }) => provider) - .sort() - ) - ).toEqual(ROUTER_PROVIDER_COVERAGE.map((row) => [...row.providers].sort())) + expectAlternateProviders( + specs, + ROUTER_PROVIDER_COVERAGE.map((row) => row.providers) + ) }, 15_000) it('lists every model the docs show with an alternate provider', async (ctx) => { From 1e7c48db571b341642634718f9003cc86a0bd4da Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 08:50:27 +0000 Subject: [PATCH 10/35] test(website): accept readonly provider fixtures --- apps/website/src/config/router-providers.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index fbece46b6a6..fd49a8cf092 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -61,7 +61,10 @@ const altProvidersSchema = z.object({ .default([]) }) -function expectAlternateProviders(specs: string[], expected: string[][]): void { +function expectAlternateProviders( + specs: string[], + expected: readonly (readonly string[])[] +): void { const actual = specs.map((spec) => altProvidersSchema .parse(JSON.parse(spec)) From c1cffc30a23a6ece6710e3bdbb024bd57bea1438 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 1 Oct 2026 03:11:03 -0700 Subject: [PATCH 11/35] test(website): discriminate router drift failure --- .../src/config/router-providers.test.ts | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index fd49a8cf092..0df93db1513 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -117,20 +117,14 @@ describe('Router provider source availability', () => { ) }) - it('detects provider drift from a successful source response', async () => { - vi.stubGlobal( - 'fetch', - vi.fn().mockResolvedValue( - new Response( - JSON.stringify({ - 'x-comfy-router-alt-providers': [{ provider: 'anthropic' }] - }) - ) - ) - ) + it('detects provider drift from a successful source response', () => { + const spec = JSON.stringify({ + 'x-comfy-router-alt-providers': [{ provider: 'anthropic' }] + }) - const spec = await fetchDocs(PROVIDERS_PAGE) - expect(() => expectAlternateProviders([spec], [['openai']])).toThrow() + expect(() => expectAlternateProviders([spec], [['openai']])).toThrow( + "expected [ [ 'anthropic' ] ] to deeply equal [ [ 'openai' ] ]" + ) }) }) From e8c12d662ab2a70635be686cf0b549dcaafd4d82 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 17:49:27 +0000 Subject: [PATCH 12/35] test(website): cover explicit CI flag values Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4153653359 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4153653388 --- apps/website/src/config/router-providers.test.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index 0df93db1513..2f775b6749c 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -19,7 +19,10 @@ const ROUTER_SCHEMAS = 'https://raw.githubusercontent.com/Comfy-Org/docs/main/router-schemas' const FETCH_ATTEMPTS = 3 const FETCH_TIMEOUT_MS = 4_000 -const isCI = () => ['1', 'true'].includes(process.env.CI?.toLowerCase() ?? '') + +function isCI(value: string | undefined = process.env.CI): boolean { + return ['1', 'true'].includes(value?.toLowerCase() ?? '') +} // The API spec and the docs are the source of truth and live outside this // repository, so the drift checks reach the network. They run in CI and skip @@ -108,6 +111,17 @@ function parseCoverageTable(markdown: string): { } describe('Router provider source availability', () => { + it.each([ + ['1', true], + ['true', true], + ['TRUE', true], + ['0', false], + ['false', false], + ['', false] + ])('classifies CI=%s as %s', (value, expected) => { + expect(isCI(value)).toBe(expected) + }) + it('fails when a required source is unavailable in CI', async () => { vi.stubEnv('CI', '1') vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline'))) From 4051e8db954dd96d7db253bc11f4c37c32eada41 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 17:49:41 +0000 Subject: [PATCH 13/35] test(website): advance source retry timers Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4153653359 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4153653370 --- apps/website/src/config/router-providers.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index 2f775b6749c..eaaaf3eaa4c 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -122,13 +122,14 @@ describe('Router provider source availability', () => { expect(isCI(value)).toBe(expected) }) - it('fails when a required source is unavailable in CI', async () => { - vi.stubEnv('CI', '1') + it('fails after retrying a source that stays unavailable', async () => { vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline'))) - await expect(fetchDocs(PROVIDERS_PAGE)).rejects.toThrow( + const result = expect(fetchDocs(PROVIDERS_PAGE)).rejects.toThrow( `Could not fetch ${PROVIDERS_PAGE}` ) + await vi.runAllTimersAsync() + await result }) it('detects provider drift from a successful source response', () => { From 65f0b46c701fbb48e181d270cf8f0455c94000ef Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 17:49:51 +0000 Subject: [PATCH 14/35] test(website): use the retry constant directly Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4153653395 --- apps/website/src/config/router-providers.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index eaaaf3eaa4c..e2c5f8b8dde 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -33,9 +33,8 @@ const skipReason = isCI() : 'checks the published API spec; runs in CI only (set CI=1 to run it here)' async function fetchDocs(url: string): Promise { - const attempts = FETCH_ATTEMPTS let lastError: unknown - for (let attempt = 0; attempt < attempts; attempt++) { + for (let attempt = 0; attempt < FETCH_ATTEMPTS; attempt++) { try { const response = await fetch(url, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) @@ -44,7 +43,7 @@ async function fetchDocs(url: string): Promise { return await response.text() } catch (error) { lastError = error - if (attempt + 1 < attempts) + if (attempt + 1 < FETCH_ATTEMPTS) await new Promise((resolve) => setTimeout(resolve, 250 * 2 ** attempt)) } } From 70e718e4e9951385b9167a665324437c67c93970 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 17:50:05 +0000 Subject: [PATCH 15/35] test(website): derive the source-test budget Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4153653401 --- apps/website/src/config/router-providers.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index e2c5f8b8dde..56e246a7240 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -19,6 +19,7 @@ const ROUTER_SCHEMAS = 'https://raw.githubusercontent.com/Comfy-Org/docs/main/router-schemas' const FETCH_ATTEMPTS = 3 const FETCH_TIMEOUT_MS = 4_000 +const SOURCE_TEST_TIMEOUT_MS = FETCH_ATTEMPTS * FETCH_TIMEOUT_MS + 3_000 function isCI(value: string | undefined = process.env.CI): boolean { return ['1', 'true'].includes(value?.toLowerCase() ?? '') @@ -162,7 +163,7 @@ describe('Router provider coverage', () => { specs, ROUTER_PROVIDER_COVERAGE.map((row) => row.providers) ) - }, 15_000) + }, SOURCE_TEST_TIMEOUT_MS) it('lists every model the docs show with an alternate provider', async (ctx) => { if (skipReason) return ctx.skip(skipReason) @@ -185,7 +186,7 @@ describe('Router provider coverage', () => { name: row.docsName ?? row.name })).sort(byDocsUrl) ) - }, 15_000) + }, SOURCE_TEST_TIMEOUT_MS) it('previews catalog models that only Comfy serves', async (ctx) => { if (skipReason) return ctx.skip(skipReason) @@ -199,5 +200,5 @@ describe('Router provider coverage', () => { expect(catalog).toContain(`[${name}](${path})`) expect(coverage).not.toContain(path) } - }, 15_000) + }, SOURCE_TEST_TIMEOUT_MS) }) From 6fb4b346b43eb0dbb2181253a28f3ab9ac95f5cc Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 17:50:19 +0000 Subject: [PATCH 16/35] test(website): name the provider mismatch case precisely Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4154360281 --- apps/website/src/config/router-providers.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index 56e246a7240..7f1f632f2f0 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -132,7 +132,7 @@ describe('Router provider source availability', () => { await result }) - it('detects provider drift from a successful source response', () => { + it('detects provider drift when the spec lists different alternate providers', () => { const spec = JSON.stringify({ 'x-comfy-router-alt-providers': [{ provider: 'anthropic' }] }) From 9393a7c3a1ea123a4719e78ef55f9c86e3cade59 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 1 Oct 2026 17:55:53 +0000 Subject: [PATCH 17/35] test(website): satisfy parameterized async lint --- .../src/config/router-providers.test.ts | 120 ++++++++++-------- 1 file changed, 66 insertions(+), 54 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index 7f1f632f2f0..331770ab05b 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -111,25 +111,25 @@ function parseCoverageTable(markdown: string): { } describe('Router provider source availability', () => { - it.each([ + for (const [value, expected] of [ ['1', true], ['true', true], ['TRUE', true], ['0', false], ['false', false], ['', false] - ])('classifies CI=%s as %s', (value, expected) => { - expect(isCI(value)).toBe(expected) - }) + ] as const) { + it(`classifies CI=${value} as ${expected}`, () => { + expect(isCI(value)).toBe(expected) + }) + } it('fails after retrying a source that stays unavailable', async () => { vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline'))) - const result = expect(fetchDocs(PROVIDERS_PAGE)).rejects.toThrow( - `Could not fetch ${PROVIDERS_PAGE}` - ) + const result = fetchDocs(PROVIDERS_PAGE) await vi.runAllTimersAsync() - await result + await expect(result).rejects.toThrow(`Could not fetch ${PROVIDERS_PAGE}`) }) it('detects provider drift when the spec lists different alternate providers', () => { @@ -151,54 +151,66 @@ describe('Router provider coverage', () => { expect(ROUTER_PROVIDER_COVERAGE_VERIFIED_AT).toMatch(/^\d{4}-\d{2}-\d{2}$/) }) - it("matches each model's alternate providers in the API spec", async (ctx) => { - if (skipReason) return ctx.skip(skipReason) - const specs = await Promise.all( - ROUTER_PROVIDER_COVERAGE.map((row) => - fetchDocs(`${ROUTER_SCHEMAS}/${row.modelId}.json`) + it( + "matches each model's alternate providers in the API spec", + async (ctx) => { + if (skipReason) return ctx.skip(skipReason) + const specs = await Promise.all( + ROUTER_PROVIDER_COVERAGE.map((row) => + fetchDocs(`${ROUTER_SCHEMAS}/${row.modelId}.json`) + ) ) - ) - expectAlternateProviders( - specs, - ROUTER_PROVIDER_COVERAGE.map((row) => row.providers) - ) - }, SOURCE_TEST_TIMEOUT_MS) + expectAlternateProviders( + specs, + ROUTER_PROVIDER_COVERAGE.map((row) => row.providers) + ) + }, + SOURCE_TEST_TIMEOUT_MS + ) - it('lists every model the docs show with an alternate provider', async (ctx) => { - if (skipReason) return ctx.skip(skipReason) - const markdown = await fetchDocs(PROVIDERS_PAGE) + it( + 'lists every model the docs show with an alternate provider', + async (ctx) => { + if (skipReason) return ctx.skip(skipReason) + const markdown = await fetchDocs(PROVIDERS_PAGE) - const docs = parseCoverageTable(markdown) - expect(docs.providers).toEqual( - ROUTER_SERVING_PROVIDERS.map((provider) => provider.name) - ) - expect(docs.rows.every((row) => row.comfy === '✓')).toBe(true) - const byDocsUrl = (a: { docsUrl: string }, b: { docsUrl: string }) => - a.docsUrl.localeCompare(b.docsUrl) - expect( - docs.rows - .map((row) => ({ docsUrl: row.docsUrl, name: row.name })) - .sort(byDocsUrl) - ).toEqual( - ROUTER_PROVIDER_COVERAGE.map((row) => ({ - docsUrl: row.docsUrl, - name: row.docsName ?? row.name - })).sort(byDocsUrl) - ) - }, SOURCE_TEST_TIMEOUT_MS) - - it('previews catalog models that only Comfy serves', async (ctx) => { - if (skipReason) return ctx.skip(skipReason) - const [catalog, coverage] = await Promise.all([ - fetchDocs(MODELS_PAGE), - fetchDocs(PROVIDERS_PAGE) - ]) - - for (const { name, docsUrl } of ROUTER_COMFY_ONLY_PREVIEW) { - const path = docsUrl.slice(DOCS_ORIGIN.length) - expect(catalog).toContain(`[${name}](${path})`) - expect(coverage).not.toContain(path) - } - }, SOURCE_TEST_TIMEOUT_MS) + const docs = parseCoverageTable(markdown) + expect(docs.providers).toEqual( + ROUTER_SERVING_PROVIDERS.map((provider) => provider.name) + ) + expect(docs.rows.every((row) => row.comfy === '✓')).toBe(true) + const byDocsUrl = (a: { docsUrl: string }, b: { docsUrl: string }) => + a.docsUrl.localeCompare(b.docsUrl) + expect( + docs.rows + .map((row) => ({ docsUrl: row.docsUrl, name: row.name })) + .sort(byDocsUrl) + ).toEqual( + ROUTER_PROVIDER_COVERAGE.map((row) => ({ + docsUrl: row.docsUrl, + name: row.docsName ?? row.name + })).sort(byDocsUrl) + ) + }, + SOURCE_TEST_TIMEOUT_MS + ) + + it( + 'previews catalog models that only Comfy serves', + async (ctx) => { + if (skipReason) return ctx.skip(skipReason) + const [catalog, coverage] = await Promise.all([ + fetchDocs(MODELS_PAGE), + fetchDocs(PROVIDERS_PAGE) + ]) + + for (const { name, docsUrl } of ROUTER_COMFY_ONLY_PREVIEW) { + const path = docsUrl.slice(DOCS_ORIGIN.length) + expect(catalog).toContain(`[${name}](${path})`) + expect(coverage).not.toContain(path) + } + }, + SOURCE_TEST_TIMEOUT_MS + ) }) From 9d96e1ec6f263adaa4c65c3db21bfa0aaf45908e Mon Sep 17 00:00:00 2001 From: bymyself Date: Fri, 2 Oct 2026 00:20:17 +0000 Subject: [PATCH 18/35] test(website): handle retry rejection before timers Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4158725416 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4158710466 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4158725450 --- .../src/config/router-providers.test.ts | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index 331770ab05b..fc5c41f1243 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -111,25 +111,26 @@ function parseCoverageTable(markdown: string): { } describe('Router provider source availability', () => { - for (const [value, expected] of [ + it.for([ ['1', true], ['true', true], ['TRUE', true], ['0', false], ['false', false], ['', false] - ] as const) { - it(`classifies CI=${value} as ${expected}`, () => { - expect(isCI(value)).toBe(expected) - }) - } + ] as const)('classifies CI=%s as %s', ([value, expected]) => { + expect(isCI(value)).toBe(expected) + }) it('fails after retrying a source that stays unavailable', async () => { vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline'))) - const result = fetchDocs(PROVIDERS_PAGE) - await vi.runAllTimersAsync() - await expect(result).rejects.toThrow(`Could not fetch ${PROVIDERS_PAGE}`) + await Promise.all([ + expect(fetchDocs(PROVIDERS_PAGE)).rejects.toThrow( + `Could not fetch ${PROVIDERS_PAGE}` + ), + vi.runAllTimersAsync() + ]) }) it('detects provider drift when the spec lists different alternate providers', () => { @@ -205,11 +206,16 @@ describe('Router provider coverage', () => { fetchDocs(PROVIDERS_PAGE) ]) - for (const { name, docsUrl } of ROUTER_COMFY_ONLY_PREVIEW) { - const path = docsUrl.slice(DOCS_ORIGIN.length) - expect(catalog).toContain(`[${name}](${path})`) - expect(coverage).not.toContain(path) - } + const missingCatalogLinks = ROUTER_COMFY_ONLY_PREVIEW.filter( + ({ name, docsUrl }) => + !catalog.includes(`[${name}](${docsUrl.slice(DOCS_ORIGIN.length)})`) + ).map(({ name }) => name) + const previewPathsInCoverage = ROUTER_COMFY_ONLY_PREVIEW.map( + ({ docsUrl }) => docsUrl.slice(DOCS_ORIGIN.length) + ).filter((path) => coverage.includes(path)) + + expect(missingCatalogLinks).toEqual([]) + expect(previewPathsInCoverage).toEqual([]) }, SOURCE_TEST_TIMEOUT_MS ) From 348f3122d2dfcd2c2d7ad5dede656b4665a7ff0d Mon Sep 17 00:00:00 2001 From: bymyself Date: Fri, 2 Oct 2026 23:19:40 +0000 Subject: [PATCH 19/35] ci(website): schedule router provider drift checks Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4162133347 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4166367309 --- .github/workflows/router-provider-drift.yaml | 33 ++++ .../src/config/router-providers.test.ts | 179 +----------------- apps/website/src/config/router-providers.ts | 8 +- apps/website/vitest.config.ts | 6 +- scripts/routerProviderDrift.test.ts | 70 +++++++ scripts/routerProviderDrift.ts | 152 +++++++++++++++ 6 files changed, 267 insertions(+), 181 deletions(-) create mode 100644 .github/workflows/router-provider-drift.yaml create mode 100644 scripts/routerProviderDrift.test.ts create mode 100644 scripts/routerProviderDrift.ts diff --git a/.github/workflows/router-provider-drift.yaml b/.github/workflows/router-provider-drift.yaml new file mode 100644 index 00000000000..dc7268ebf0e --- /dev/null +++ b/.github/workflows/router-provider-drift.yaml @@ -0,0 +1,33 @@ +# Published Router schemas and docs are external state, so this alarm runs on a +# schedule instead of gating unrelated pull requests. +name: 'Router Provider Drift' + +on: + schedule: + - cron: '37 13 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + drift-check: + if: github.repository == 'Comfy-Org/ComfyUI_frontend' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + - uses: actions/setup-node@v6 + with: + node-version-file: '.nvmrc' + cache: pnpm + - run: pnpm install --frozen-lockfile + - name: Compare checked-in coverage with published sources + run: pnpm exec tsx scripts/routerProviderDrift.ts diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index fc5c41f1243..9176e70aa57 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -1,63 +1,13 @@ // @vitest-environment node import { z } from 'astro/zod' -import { describe, expect, it, vi } from 'vitest' +import { describe, expect, it } from 'vitest' import { - ROUTER_COMFY_ONLY_PREVIEW, ROUTER_PROVIDER_COVERAGE, - ROUTER_PROVIDER_COVERAGE_VERIFIED_AT, - ROUTER_SERVING_PROVIDERS + ROUTER_PROVIDER_COVERAGE_VERIFIED_AT } from './router-providers' -const DOCS_ORIGIN = 'https://docs.comfy.org' -const ROUTER_DOCS = `${DOCS_ORIGIN}/development/comfy-router` -const PROVIDERS_PAGE = `${ROUTER_DOCS}/providers.md` -const MODELS_PAGE = `${ROUTER_DOCS}/models.md` -// The per-model API spec, as the backend publishes it for the docs site. -const ROUTER_SCHEMAS = - 'https://raw.githubusercontent.com/Comfy-Org/docs/main/router-schemas' -const FETCH_ATTEMPTS = 3 -const FETCH_TIMEOUT_MS = 4_000 -const SOURCE_TEST_TIMEOUT_MS = FETCH_ATTEMPTS * FETCH_TIMEOUT_MS + 3_000 - -function isCI(value: string | undefined = process.env.CI): boolean { - return ['1', 'true'].includes(value?.toLowerCase() ?? '') -} - -// The API spec and the docs are the source of truth and live outside this -// repository, so the drift checks reach the network. They run in CI and skip -// elsewhere. CI treats an unavailable source as a failure so missing coverage -// cannot produce a green build. -const skipReason = isCI() - ? null - : 'checks the published API spec; runs in CI only (set CI=1 to run it here)' - -async function fetchDocs(url: string): Promise { - let lastError: unknown - for (let attempt = 0; attempt < FETCH_ATTEMPTS; attempt++) { - try { - const response = await fetch(url, { - signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) - }) - if (!response.ok) throw new Error(`${url} responded ${response.status}`) - return await response.text() - } catch (error) { - lastError = error - if (attempt + 1 < FETCH_ATTEMPTS) - await new Promise((resolve) => setTimeout(resolve, 250 * 2 ** attempt)) - } - } - throw new Error(`Could not fetch ${url}`, { cause: lastError }) -} - -interface DocsCoverageRow { - name: string - docsUrl: string - comfy: string - providers: string[] -} - const altProvidersSchema = z.object({ 'x-comfy-router-alt-providers': z .array(z.object({ provider: z.string() })) @@ -77,62 +27,7 @@ function expectAlternateProviders( expect(actual).toEqual(expected.map((providers) => [...providers].sort())) } -/** Reads the "Provider coverage" table from the docs page's Markdown. */ -function parseCoverageTable(markdown: string): { - providers: string[] - rows: DocsCoverageRow[] -} { - const section = markdown.slice(markdown.indexOf('## Provider coverage')) - const lines = section - .split('\n') - .map((line) => line.trim()) - .filter((line) => line.startsWith('|')) - const cells = (line: string) => - line - .split('|') - .slice(1, -1) - .map((cell) => cell.trim()) - const [header, , ...body] = lines - const columns = cells(header).map((column) => column.replaceAll('**', '')) - expect(columns.slice(0, 2)).toEqual(['Model / provider', 'Comfy (default)']) - const providers = columns.slice(2) - const rows = body.map((line) => { - const [model, comfy, ...routes] = cells(line) - const link = /^\[(.+)\]\((.+)\)$/.exec(model) - if (!link) throw new Error(`Unexpected model cell: ${model}`) - return { - name: link[1], - docsUrl: `${DOCS_ORIGIN}${link[2]}`, - comfy, - providers: providers.filter((_, index) => routes[index] !== '-') - } - }) - return { providers, rows } -} - -describe('Router provider source availability', () => { - it.for([ - ['1', true], - ['true', true], - ['TRUE', true], - ['0', false], - ['false', false], - ['', false] - ] as const)('classifies CI=%s as %s', ([value, expected]) => { - expect(isCI(value)).toBe(expected) - }) - - it('fails after retrying a source that stays unavailable', async () => { - vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline'))) - - await Promise.all([ - expect(fetchDocs(PROVIDERS_PAGE)).rejects.toThrow( - `Could not fetch ${PROVIDERS_PAGE}` - ), - vi.runAllTimersAsync() - ]) - }) - +describe('Router provider fixtures', () => { it('detects provider drift when the spec lists different alternate providers', () => { const spec = JSON.stringify({ 'x-comfy-router-alt-providers': [{ provider: 'anthropic' }] @@ -151,72 +46,4 @@ describe('Router provider coverage', () => { expect(new Set(names).size).toBe(names.length) expect(ROUTER_PROVIDER_COVERAGE_VERIFIED_AT).toMatch(/^\d{4}-\d{2}-\d{2}$/) }) - - it( - "matches each model's alternate providers in the API spec", - async (ctx) => { - if (skipReason) return ctx.skip(skipReason) - const specs = await Promise.all( - ROUTER_PROVIDER_COVERAGE.map((row) => - fetchDocs(`${ROUTER_SCHEMAS}/${row.modelId}.json`) - ) - ) - - expectAlternateProviders( - specs, - ROUTER_PROVIDER_COVERAGE.map((row) => row.providers) - ) - }, - SOURCE_TEST_TIMEOUT_MS - ) - - it( - 'lists every model the docs show with an alternate provider', - async (ctx) => { - if (skipReason) return ctx.skip(skipReason) - const markdown = await fetchDocs(PROVIDERS_PAGE) - - const docs = parseCoverageTable(markdown) - expect(docs.providers).toEqual( - ROUTER_SERVING_PROVIDERS.map((provider) => provider.name) - ) - expect(docs.rows.every((row) => row.comfy === '✓')).toBe(true) - const byDocsUrl = (a: { docsUrl: string }, b: { docsUrl: string }) => - a.docsUrl.localeCompare(b.docsUrl) - expect( - docs.rows - .map((row) => ({ docsUrl: row.docsUrl, name: row.name })) - .sort(byDocsUrl) - ).toEqual( - ROUTER_PROVIDER_COVERAGE.map((row) => ({ - docsUrl: row.docsUrl, - name: row.docsName ?? row.name - })).sort(byDocsUrl) - ) - }, - SOURCE_TEST_TIMEOUT_MS - ) - - it( - 'previews catalog models that only Comfy serves', - async (ctx) => { - if (skipReason) return ctx.skip(skipReason) - const [catalog, coverage] = await Promise.all([ - fetchDocs(MODELS_PAGE), - fetchDocs(PROVIDERS_PAGE) - ]) - - const missingCatalogLinks = ROUTER_COMFY_ONLY_PREVIEW.filter( - ({ name, docsUrl }) => - !catalog.includes(`[${name}](${docsUrl.slice(DOCS_ORIGIN.length)})`) - ).map(({ name }) => name) - const previewPathsInCoverage = ROUTER_COMFY_ONLY_PREVIEW.map( - ({ docsUrl }) => docsUrl.slice(DOCS_ORIGIN.length) - ).filter((path) => coverage.includes(path)) - - expect(missingCatalogLinks).toEqual([]) - expect(previewPathsInCoverage).toEqual([]) - }, - SOURCE_TEST_TIMEOUT_MS - ) }) diff --git a/apps/website/src/config/router-providers.ts b/apps/website/src/config/router-providers.ts index 0e6a84f08ed..586d0764cbd 100644 --- a/apps/website/src/config/router-providers.ts +++ b/apps/website/src/config/router-providers.ts @@ -4,10 +4,10 @@ * Every Router model runs through Comfy's default route. The rows below are the * only ones that also accept `model_provider`. They mirror the API spec: each * model's `x-comfy-router-alt-providers`, published per model under - * `router-schemas/` in the public Comfy-Org/docs repository. Reading the live - * spec needs an API key, so the rows are copied by hand; - * `router-providers.test.ts` compares them with the published spec and fails - * the day they drift. Update `ROUTER_PROVIDER_COVERAGE_VERIFIED_AT` with them. + * `router-schemas/` in the public Comfy-Org/docs repository. The rows are + * copied by hand; the scheduled Router Provider Drift workflow compares them + * with the published spec. Update `ROUTER_PROVIDER_COVERAGE_VERIFIED_AT` with + * them. */ export const ROUTER_SERVING_PROVIDERS = [ diff --git a/apps/website/vitest.config.ts b/apps/website/vitest.config.ts index 8bbe9d8fdf4..7f86417647a 100644 --- a/apps/website/vitest.config.ts +++ b/apps/website/vitest.config.ts @@ -30,7 +30,11 @@ export default defineConfig({ }, include: ['src/**/*.{test,spec}.ts', 'scripts/**/*.{test,spec}.ts'], globals: false, - setupFiles: ['../../vitest.timer.setup.ts', './src/test/setup.ts'], + setupFiles: [ + '../../vitest.network.setup.ts', + '../../vitest.timer.setup.ts', + './src/test/setup.ts' + ], coverage: { provider: 'v8', reporter: ['text', 'lcov'], diff --git a/scripts/routerProviderDrift.test.ts b/scripts/routerProviderDrift.test.ts new file mode 100644 index 00000000000..9e0d3f19a55 --- /dev/null +++ b/scripts/routerProviderDrift.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it, vi } from 'vitest' + +import { + alternateProviders, + fetchText, + parseCoverageTable +} from './routerProviderDrift' + +describe('fetchText', () => { + it('recovers when a source fails once and then responds', async () => { + const fetchImpl = vi + .fn() + .mockRejectedValueOnce(new Error('offline')) + .mockResolvedValueOnce(new Response('providers')) + const sleep = vi.fn(async () => {}) + + await expect( + fetchText('https://example.com', { fetchImpl, sleep }) + ).resolves.toBe('providers') + expect(fetchImpl).toHaveBeenCalledTimes(2) + expect(sleep).toHaveBeenCalledOnce() + }) + + it('reports an unavailable source after every attempt', async () => { + const fetchImpl = vi + .fn() + .mockRejectedValue(new Error('offline')) + + await expect( + fetchText('https://example.com', { + fetchImpl, + sleep: async () => {} + }) + ).rejects.toThrow('Could not fetch https://example.com') + expect(fetchImpl).toHaveBeenCalledTimes(3) + }) +}) + +it('reads alternate providers from a published model schema', () => { + expect( + alternateProviders( + JSON.stringify({ + 'x-comfy-router-alt-providers': [ + { provider: 'fal' }, + { provider: 'runware' } + ] + }) + ) + ).toEqual(['fal', 'runware']) +}) + +it('reads the provider coverage table from Markdown', () => { + const markdown = ` +## Provider coverage +| **Model / provider** | **Comfy (default)** | **fal** | +| --- | --- | --- | +| [Model](/models/model/code) | ✓ | ✓ | +` + + expect(parseCoverageTable(markdown)).toEqual({ + providers: ['fal'], + rows: [ + { + name: 'Model', + docsUrl: 'https://docs.comfy.org/models/model/code', + comfy: '✓' + } + ] + }) +}) diff --git a/scripts/routerProviderDrift.ts b/scripts/routerProviderDrift.ts new file mode 100644 index 00000000000..a3dd1a4fb43 --- /dev/null +++ b/scripts/routerProviderDrift.ts @@ -0,0 +1,152 @@ +import assert from 'node:assert/strict' +import { pathToFileURL } from 'node:url' + +import { + ROUTER_COMFY_ONLY_PREVIEW, + ROUTER_PROVIDER_COVERAGE, + ROUTER_SERVING_PROVIDERS +} from '../apps/website/src/config/router-providers' + +const DOCS_ORIGIN = 'https://docs.comfy.org' +const ROUTER_DOCS = `${DOCS_ORIGIN}/development/comfy-router` +const PROVIDERS_PAGE = `${ROUTER_DOCS}/providers.md` +const MODELS_PAGE = `${ROUTER_DOCS}/models.md` +const ROUTER_SCHEMAS = + 'https://raw.githubusercontent.com/Comfy-Org/docs/main/router-schemas' + +interface FetchTextOptions { + attempts?: number + fetchImpl?: typeof fetch + sleep?: (milliseconds: number) => Promise + timeoutMs?: number +} + +interface DocsCoverageRow { + name: string + docsUrl: string + comfy: string +} + +export async function fetchText( + url: string, + { + attempts = 3, + fetchImpl = fetch, + sleep = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)), + timeoutMs = 4_000 + }: FetchTextOptions = {} +): Promise { + let lastError: unknown + for (let attempt = 0; attempt < attempts; attempt++) { + try { + const response = await fetchImpl(url, { + signal: AbortSignal.timeout(timeoutMs) + }) + if (!response.ok) throw new Error(`${url} responded ${response.status}`) + return await response.text() + } catch (error) { + lastError = error + if (attempt + 1 < attempts) await sleep(250 * 2 ** attempt) + } + } + throw new Error(`Could not fetch ${url}`, { cause: lastError }) +} + +export function alternateProviders(spec: string): string[] { + const parsed: unknown = JSON.parse(spec) + assert.ok(parsed && typeof parsed === 'object') + const providers = Reflect.get(parsed, 'x-comfy-router-alt-providers') + assert.ok(Array.isArray(providers)) + return providers.map((entry: unknown) => { + assert.ok(entry && typeof entry === 'object') + const provider = Reflect.get(entry, 'provider') + assert.equal(typeof provider, 'string') + return provider + }) +} + +export function parseCoverageTable(markdown: string): { + providers: string[] + rows: DocsCoverageRow[] +} { + const section = markdown.slice(markdown.indexOf('## Provider coverage')) + const lines = section + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.startsWith('|')) + const cells = (line: string) => + line + .split('|') + .slice(1, -1) + .map((cell) => cell.trim()) + const [header, , ...body] = lines + assert.ok(header, 'Provider coverage table is missing') + const columns = cells(header).map((column) => column.replaceAll('**', '')) + assert.deepEqual(columns.slice(0, 2), ['Model / provider', 'Comfy (default)']) + return { + providers: columns.slice(2), + rows: body.map((line) => { + const [model, comfy] = cells(line) + const link = /^\[(.+)\]\((.+)\)$/.exec(model) + assert.ok(link, `Unexpected model cell: ${model}`) + return { name: link[1], docsUrl: `${DOCS_ORIGIN}${link[2]}`, comfy } + }) + } +} + +export async function checkRouterProviderDrift(): Promise { + const [coverageMarkdown, catalogMarkdown, ...specs] = await Promise.all([ + fetchText(PROVIDERS_PAGE), + fetchText(MODELS_PAGE), + ...ROUTER_PROVIDER_COVERAGE.map((row) => + fetchText(`${ROUTER_SCHEMAS}/${row.modelId}.json`) + ) + ]) + assert.deepEqual( + specs.map((spec) => alternateProviders(spec).sort()), + ROUTER_PROVIDER_COVERAGE.map((row) => [...row.providers].sort()) + ) + + const docs = parseCoverageTable(coverageMarkdown) + assert.deepEqual( + docs.providers, + ROUTER_SERVING_PROVIDERS.map((provider) => provider.name) + ) + assert.ok(docs.rows.every((row) => row.comfy === '✓')) + const byDocsUrl = (a: { docsUrl: string }, b: { docsUrl: string }) => + a.docsUrl.localeCompare(b.docsUrl) + assert.deepEqual( + docs.rows.map(({ docsUrl, name }) => ({ docsUrl, name })).sort(byDocsUrl), + ROUTER_PROVIDER_COVERAGE.map((row) => ({ + docsUrl: row.docsUrl, + name: row.docsName ?? row.name + })).sort(byDocsUrl) + ) + + const missingCatalogLinks = ROUTER_COMFY_ONLY_PREVIEW.filter( + ({ name, docsUrl }) => + !catalogMarkdown.includes( + `[${name}](${docsUrl.slice(DOCS_ORIGIN.length)})` + ) + ).map(({ name }) => name) + const previewPathsInCoverage = ROUTER_COMFY_ONLY_PREVIEW.map(({ docsUrl }) => + docsUrl.slice(DOCS_ORIGIN.length) + ).filter((path) => coverageMarkdown.includes(path)) + assert.deepEqual(missingCatalogLinks, []) + assert.deepEqual(previewPathsInCoverage, []) +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + checkRouterProviderDrift() + .then(() => { + console.log('Router provider coverage matches the published sources.') + }) + .catch((error: unknown) => { + console.error(error) + process.exitCode = 1 + }) +} From 29fe9472e4a4de91e69a2534cf6c51f658174648 Mon Sep 17 00:00:00 2001 From: bymyself Date: Fri, 2 Oct 2026 23:24:32 +0000 Subject: [PATCH 20/35] fix(ci): keep drift script within website package --- .github/workflows/router-provider-drift.yaml | 2 +- {scripts => apps/website/scripts}/routerProviderDrift.test.ts | 0 {scripts => apps/website/scripts}/routerProviderDrift.ts | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename {scripts => apps/website/scripts}/routerProviderDrift.test.ts (100%) rename {scripts => apps/website/scripts}/routerProviderDrift.ts (98%) diff --git a/.github/workflows/router-provider-drift.yaml b/.github/workflows/router-provider-drift.yaml index dc7268ebf0e..d797b438145 100644 --- a/.github/workflows/router-provider-drift.yaml +++ b/.github/workflows/router-provider-drift.yaml @@ -30,4 +30,4 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - name: Compare checked-in coverage with published sources - run: pnpm exec tsx scripts/routerProviderDrift.ts + run: pnpm --filter @comfyorg/website exec tsx scripts/routerProviderDrift.ts diff --git a/scripts/routerProviderDrift.test.ts b/apps/website/scripts/routerProviderDrift.test.ts similarity index 100% rename from scripts/routerProviderDrift.test.ts rename to apps/website/scripts/routerProviderDrift.test.ts diff --git a/scripts/routerProviderDrift.ts b/apps/website/scripts/routerProviderDrift.ts similarity index 98% rename from scripts/routerProviderDrift.ts rename to apps/website/scripts/routerProviderDrift.ts index a3dd1a4fb43..61fe17bcdc7 100644 --- a/scripts/routerProviderDrift.ts +++ b/apps/website/scripts/routerProviderDrift.ts @@ -5,7 +5,7 @@ import { ROUTER_COMFY_ONLY_PREVIEW, ROUTER_PROVIDER_COVERAGE, ROUTER_SERVING_PROVIDERS -} from '../apps/website/src/config/router-providers' +} from '../src/config/router-providers' const DOCS_ORIGIN = 'https://docs.comfy.org' const ROUTER_DOCS = `${DOCS_ORIGIN}/development/comfy-router` From 833d70958f19945796992954cb6ce053a3686380 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 00:09:17 +0000 Subject: [PATCH 21/35] fix(website): satisfy router drift lint checks --- apps/website/scripts/routerProviderDrift.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/website/scripts/routerProviderDrift.ts b/apps/website/scripts/routerProviderDrift.ts index 61fe17bcdc7..122024eac79 100644 --- a/apps/website/scripts/routerProviderDrift.ts +++ b/apps/website/scripts/routerProviderDrift.ts @@ -95,7 +95,7 @@ export function parseCoverageTable(markdown: string): { } } -export async function checkRouterProviderDrift(): Promise { +async function checkRouterProviderDrift(): Promise { const [coverageMarkdown, catalogMarkdown, ...specs] = await Promise.all([ fetchText(PROVIDERS_PAGE), fetchText(MODELS_PAGE), @@ -143,7 +143,7 @@ if ( ) { checkRouterProviderDrift() .then(() => { - console.log('Router provider coverage matches the published sources.') + console.warn('Router provider coverage matches the published sources.') }) .catch((error: unknown) => { console.error(error) From 93441faf4acb0546957d9d1e984cb263821b9b2b Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 02:48:31 +0000 Subject: [PATCH 22/35] test(website): close router drift review gaps --- .github/workflows/router-provider-drift.yaml | 72 +++++++++++++++++-- .../scripts/routerProviderDrift.test.ts | 63 ++++++++++++++++ apps/website/scripts/routerProviderDrift.ts | 54 +++++++------- .../src/config/router-providers.test.ts | 32 --------- 4 files changed, 156 insertions(+), 65 deletions(-) diff --git a/.github/workflows/router-provider-drift.yaml b/.github/workflows/router-provider-drift.yaml index d797b438145..f8fbd4fe4b3 100644 --- a/.github/workflows/router-provider-drift.yaml +++ b/.github/workflows/router-provider-drift.yaml @@ -9,6 +9,11 @@ on: permissions: contents: read + issues: write + +env: + ISSUE_LABEL: 'area:testing' + ISSUE_TITLE: 'Router provider coverage has drifted from published sources' concurrency: group: ${{ github.workflow }} @@ -23,11 +28,66 @@ jobs: - uses: actions/checkout@v7 with: persist-credentials: false - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - - uses: actions/setup-node@v6 - with: - node-version-file: '.nvmrc' - cache: pnpm - - run: pnpm install --frozen-lockfile + - uses: ./.github/actions/setup-frontend - name: Compare checked-in coverage with published sources run: pnpm --filter @comfyorg/website exec tsx scripts/routerProviderDrift.ts + - name: File or update the drift issue + if: failure() + uses: actions/github-script@v9 + with: + github-token: ${{ github.token }} + script: | + const title = process.env.ISSUE_TITLE + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + const body = `The scheduled [Router Provider Drift](${runUrl}) check failed. The checked-in Router provider coverage may no longer match the published documentation or schemas, or a published source may be unavailable. See the run for the exact failure. This issue is maintained by \`.github/workflows/router-provider-drift.yaml\` and closes itself after a passing run.` + const openIssues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: process.env.ISSUE_LABEL, + per_page: 100 + }) + const existing = openIssues.find((issue) => issue.title === title) + if (existing) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body + }) + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + labels: [process.env.ISSUE_LABEL] + }) + } + - name: Close the drift issue after recovery + if: success() + uses: actions/github-script@v9 + with: + github-token: ${{ github.token }} + script: | + const openIssues = await github.paginate(github.rest.issues.listForRepo, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: process.env.ISSUE_LABEL, + per_page: 100 + }) + const existing = openIssues.find((issue) => issue.title === process.env.ISSUE_TITLE) + if (!existing) return + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body: 'Router provider coverage matches the published sources again. Closing.' + }) + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + state: 'closed' + }) diff --git a/apps/website/scripts/routerProviderDrift.test.ts b/apps/website/scripts/routerProviderDrift.test.ts index 9e0d3f19a55..5c93bfc5943 100644 --- a/apps/website/scripts/routerProviderDrift.test.ts +++ b/apps/website/scripts/routerProviderDrift.test.ts @@ -1,11 +1,19 @@ import { describe, expect, it, vi } from 'vitest' +import { + ROUTER_COMFY_ONLY_PREVIEW, + ROUTER_PROVIDER_COVERAGE, + ROUTER_SERVING_PROVIDERS +} from '../src/config/router-providers' import { alternateProviders, + checkRouterProviderDrift, fetchText, parseCoverageTable } from './routerProviderDrift' +const DOCS_ORIGIN = 'https://docs.comfy.org' + describe('fetchText', () => { it('recovers when a source fails once and then responds', async () => { const fetchImpl = vi @@ -34,6 +42,19 @@ describe('fetchText', () => { ).rejects.toThrow('Could not fetch https://example.com') expect(fetchImpl).toHaveBeenCalledTimes(3) }) + + it('reports an HTTP error as an unavailable source', async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response('maintenance', { status: 503 })) + + await expect( + fetchText('https://example.com', { + fetchImpl, + sleep: async () => {} + }) + ).rejects.toThrow('Could not fetch https://example.com') + }) }) it('reads alternate providers from a published model schema', () => { @@ -68,3 +89,45 @@ it('reads the provider coverage table from Markdown', () => { ] }) }) + +it('detects drift with controlled published-source fixtures', async () => { + const coverageRows = ROUTER_PROVIDER_COVERAGE.map( + ({ docsName, docsUrl, name }) => + `| [${docsName ?? name}](${docsUrl.slice(DOCS_ORIGIN.length)}) | ✓ |` + ).join('\n') + const coverage = ` +## Provider coverage +| **Model / provider** | **Comfy (default)** | ${ROUTER_SERVING_PROVIDERS.map(({ name }) => `**${name}**`).join(' | ')} | +| --- | --- | --- | +${coverageRows} +` + const catalog = ROUTER_COMFY_ONLY_PREVIEW.map( + ({ docsUrl, name }) => `[${name}](${docsUrl.slice(DOCS_ORIGIN.length)})` + ).join('\n') + const fetchImpl = vi.fn(async (input) => { + const url = String(input) + if (url.endsWith('/providers.md')) return new Response(coverage) + if (url.endsWith('/models.md')) return new Response(catalog) + const row = ROUTER_PROVIDER_COVERAGE.find(({ modelId }) => + url.endsWith(`/${modelId}.json`) + ) + return new Response( + JSON.stringify({ + 'x-comfy-router-alt-providers': (row?.providers ?? []).map( + (provider) => ({ provider }) + ) + }) + ) + }) + + await expect( + checkRouterProviderDrift({ fetchImpl, sleep: async () => {} }) + ).resolves.toBeUndefined() + + fetchImpl.mockImplementationOnce(async () => + Promise.resolve(new Response(coverage.replace('Model / provider', 'Drift'))) + ) + await expect( + checkRouterProviderDrift({ fetchImpl, sleep: async () => {} }) + ).rejects.toThrow() +}) diff --git a/apps/website/scripts/routerProviderDrift.ts b/apps/website/scripts/routerProviderDrift.ts index 122024eac79..bad34eea1f7 100644 --- a/apps/website/scripts/routerProviderDrift.ts +++ b/apps/website/scripts/routerProviderDrift.ts @@ -1,11 +1,13 @@ import assert from 'node:assert/strict' -import { pathToFileURL } from 'node:url' + +import { z } from 'zod' import { ROUTER_COMFY_ONLY_PREVIEW, ROUTER_PROVIDER_COVERAGE, ROUTER_SERVING_PROVIDERS } from '../src/config/router-providers' +import { isDirectExecution } from './script-entry-point' const DOCS_ORIGIN = 'https://docs.comfy.org' const ROUTER_DOCS = `${DOCS_ORIGIN}/development/comfy-router` @@ -15,12 +17,22 @@ const ROUTER_SCHEMAS = 'https://raw.githubusercontent.com/Comfy-Org/docs/main/router-schemas' interface FetchTextOptions { - attempts?: number fetchImpl?: typeof fetch sleep?: (milliseconds: number) => Promise - timeoutMs?: number } +const FETCH_ATTEMPTS = 3 +const FETCH_TIMEOUT_MS = 4_000 +const alternateProvidersSchema = z + .object({ + 'x-comfy-router-alt-providers': z + .array(z.object({ provider: z.string() })) + .default([]) + }) + .transform((schema) => + schema['x-comfy-router-alt-providers'].map(({ provider }) => provider) + ) + interface DocsCoverageRow { name: string docsUrl: string @@ -30,40 +42,29 @@ interface DocsCoverageRow { export async function fetchText( url: string, { - attempts = 3, fetchImpl = fetch, sleep = (milliseconds) => - new Promise((resolve) => setTimeout(resolve, milliseconds)), - timeoutMs = 4_000 + new Promise((resolve) => setTimeout(resolve, milliseconds)) }: FetchTextOptions = {} ): Promise { let lastError: unknown - for (let attempt = 0; attempt < attempts; attempt++) { + for (let attempt = 0; attempt < FETCH_ATTEMPTS; attempt++) { try { const response = await fetchImpl(url, { - signal: AbortSignal.timeout(timeoutMs) + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) }) if (!response.ok) throw new Error(`${url} responded ${response.status}`) return await response.text() } catch (error) { lastError = error - if (attempt + 1 < attempts) await sleep(250 * 2 ** attempt) + if (attempt + 1 < FETCH_ATTEMPTS) await sleep(250 * 2 ** attempt) } } throw new Error(`Could not fetch ${url}`, { cause: lastError }) } export function alternateProviders(spec: string): string[] { - const parsed: unknown = JSON.parse(spec) - assert.ok(parsed && typeof parsed === 'object') - const providers = Reflect.get(parsed, 'x-comfy-router-alt-providers') - assert.ok(Array.isArray(providers)) - return providers.map((entry: unknown) => { - assert.ok(entry && typeof entry === 'object') - const provider = Reflect.get(entry, 'provider') - assert.equal(typeof provider, 'string') - return provider - }) + return alternateProvidersSchema.parse(JSON.parse(spec)) } export function parseCoverageTable(markdown: string): { @@ -95,12 +96,14 @@ export function parseCoverageTable(markdown: string): { } } -async function checkRouterProviderDrift(): Promise { +export async function checkRouterProviderDrift( + options: FetchTextOptions = {} +): Promise { const [coverageMarkdown, catalogMarkdown, ...specs] = await Promise.all([ - fetchText(PROVIDERS_PAGE), - fetchText(MODELS_PAGE), + fetchText(PROVIDERS_PAGE, options), + fetchText(MODELS_PAGE, options), ...ROUTER_PROVIDER_COVERAGE.map((row) => - fetchText(`${ROUTER_SCHEMAS}/${row.modelId}.json`) + fetchText(`${ROUTER_SCHEMAS}/${row.modelId}.json`, options) ) ]) assert.deepEqual( @@ -137,10 +140,7 @@ async function checkRouterProviderDrift(): Promise { assert.deepEqual(previewPathsInCoverage, []) } -if ( - process.argv[1] && - import.meta.url === pathToFileURL(process.argv[1]).href -) { +if (isDirectExecution(process.argv[1], import.meta.filename)) { checkRouterProviderDrift() .then(() => { console.warn('Router provider coverage matches the published sources.') diff --git a/apps/website/src/config/router-providers.test.ts b/apps/website/src/config/router-providers.test.ts index 9176e70aa57..2876acc8f5e 100644 --- a/apps/website/src/config/router-providers.test.ts +++ b/apps/website/src/config/router-providers.test.ts @@ -1,6 +1,5 @@ // @vitest-environment node -import { z } from 'astro/zod' import { describe, expect, it } from 'vitest' import { @@ -8,37 +7,6 @@ import { ROUTER_PROVIDER_COVERAGE_VERIFIED_AT } from './router-providers' -const altProvidersSchema = z.object({ - 'x-comfy-router-alt-providers': z - .array(z.object({ provider: z.string() })) - .default([]) -}) - -function expectAlternateProviders( - specs: string[], - expected: readonly (readonly string[])[] -): void { - const actual = specs.map((spec) => - altProvidersSchema - .parse(JSON.parse(spec)) - ['x-comfy-router-alt-providers'].map(({ provider }) => provider) - .sort() - ) - expect(actual).toEqual(expected.map((providers) => [...providers].sort())) -} - -describe('Router provider fixtures', () => { - it('detects provider drift when the spec lists different alternate providers', () => { - const spec = JSON.stringify({ - 'x-comfy-router-alt-providers': [{ provider: 'anthropic' }] - }) - - expect(() => expectAlternateProviders([spec], [['openai']])).toThrow( - "expected [ [ 'anthropic' ] ] to deeply equal [ [ 'openai' ] ]" - ) - }) -}) - describe('Router provider coverage', () => { it('lists each model once, alphabetically', () => { const names = ROUTER_PROVIDER_COVERAGE.map((row) => row.name) From eb7bc549e76b9dc0c84fadbe8cfb9fb086a3290a Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 06:37:03 +0000 Subject: [PATCH 23/35] chore(website): use kebab-case for drift scripts Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4170677030 --- ...outerProviderDrift.test.ts => router-provider-drift.test.ts} | 2 +- .../{routerProviderDrift.ts => router-provider-drift.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename apps/website/scripts/{routerProviderDrift.test.ts => router-provider-drift.test.ts} (99%) rename apps/website/scripts/{routerProviderDrift.ts => router-provider-drift.ts} (100%) diff --git a/apps/website/scripts/routerProviderDrift.test.ts b/apps/website/scripts/router-provider-drift.test.ts similarity index 99% rename from apps/website/scripts/routerProviderDrift.test.ts rename to apps/website/scripts/router-provider-drift.test.ts index 5c93bfc5943..8a89383b11a 100644 --- a/apps/website/scripts/routerProviderDrift.test.ts +++ b/apps/website/scripts/router-provider-drift.test.ts @@ -10,7 +10,7 @@ import { checkRouterProviderDrift, fetchText, parseCoverageTable -} from './routerProviderDrift' +} from './router-provider-drift' const DOCS_ORIGIN = 'https://docs.comfy.org' diff --git a/apps/website/scripts/routerProviderDrift.ts b/apps/website/scripts/router-provider-drift.ts similarity index 100% rename from apps/website/scripts/routerProviderDrift.ts rename to apps/website/scripts/router-provider-drift.ts From f29e4cb5b256909e1e9d75bf76393970522dcbb1 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 06:37:03 +0000 Subject: [PATCH 24/35] chore(ci): name router drift workflow consistently Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4170677055 --- ...uter-provider-drift.yaml => ci-router-provider-drift.yaml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{router-provider-drift.yaml => ci-router-provider-drift.yaml} (94%) diff --git a/.github/workflows/router-provider-drift.yaml b/.github/workflows/ci-router-provider-drift.yaml similarity index 94% rename from .github/workflows/router-provider-drift.yaml rename to .github/workflows/ci-router-provider-drift.yaml index f8fbd4fe4b3..10de354ca14 100644 --- a/.github/workflows/router-provider-drift.yaml +++ b/.github/workflows/ci-router-provider-drift.yaml @@ -30,7 +30,7 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-frontend - name: Compare checked-in coverage with published sources - run: pnpm --filter @comfyorg/website exec tsx scripts/routerProviderDrift.ts + run: pnpm --filter @comfyorg/website check:router-provider-drift - name: File or update the drift issue if: failure() uses: actions/github-script@v9 @@ -39,7 +39,7 @@ jobs: script: | const title = process.env.ISSUE_TITLE const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` - const body = `The scheduled [Router Provider Drift](${runUrl}) check failed. The checked-in Router provider coverage may no longer match the published documentation or schemas, or a published source may be unavailable. See the run for the exact failure. This issue is maintained by \`.github/workflows/router-provider-drift.yaml\` and closes itself after a passing run.` + const body = `The scheduled [Router Provider Drift](${runUrl}) check failed. The checked-in Router provider coverage may no longer match the published documentation or schemas, or a published source may be unavailable. See the run for the exact failure. This issue is maintained by \`.github/workflows/ci-router-provider-drift.yaml\` and closes itself after a passing run.` const openIssues = await github.paginate(github.rest.issues.listForRepo, { owner: context.repo.owner, repo: context.repo.repo, From 457212e7defe44a977ae54d5129ad26b30823ba0 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 06:37:03 +0000 Subject: [PATCH 25/35] docs(website): document router drift command Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#pullrequestreview-5383739848 --- apps/website/README.md | 1 + apps/website/package.json | 1 + apps/website/scripts/README.md | 10 ++++++++++ 3 files changed, 12 insertions(+) diff --git a/apps/website/README.md b/apps/website/README.md index 0f7082af6ca..ad40104109a 100644 --- a/apps/website/README.md +++ b/apps/website/README.md @@ -508,6 +508,7 @@ the hosted script once, and renders the documented embed container. - `pnpm build:e2e` — indexable build to `dist/`, the one e2e and screenshots run against - `pnpm typecheck` — `astro check` - `pnpm test:unit` — Vitest unit tests +- `pnpm check:router-provider-drift` — compare Router coverage with published sources (requires network access) - `pnpm test:e2e` — Playwright E2E tests (requires `pnpm build:e2e` first) - `pnpm ashby:refresh-snapshot` — refresh the committed careers snapshot - `pnpm cloud-nodes:refresh-snapshot` — refresh the committed cloud nodes snapshot diff --git a/apps/website/package.json b/apps/website/package.json index aefb63b92cd..35f67f9bc7c 100644 --- a/apps/website/package.json +++ b/apps/website/package.json @@ -15,6 +15,7 @@ "typecheck": "astro check && tsx ../../scripts/vue-tsc-native.ts --noEmit -p tsconfig.json", "test:unit": "vitest run", "test:router-models": "tsx ./scripts/test-router-models.ts", + "check:router-provider-drift": "tsx ./scripts/router-provider-drift.ts", "test:router-validation": "vitest run scripts/router-model-validation.test.ts src/config/workshop-input-validation.test.ts src/config/workshop-media-validation.test.ts src/config/workshop-media-metadata.test.ts", "test:workflow-pages": "tsx ./scripts/test-workflow-pages.ts", "test:workshop-upload": "tsx ./scripts/test-workshop-upload.ts", diff --git a/apps/website/scripts/README.md b/apps/website/scripts/README.md index 87aa403876a..2b3e6025fc0 100644 --- a/apps/website/scripts/README.md +++ b/apps/website/scripts/README.md @@ -1,5 +1,15 @@ # Website Scripts +## `router-provider-drift.ts` + +Compares the checked-in Router provider coverage with the published schemas and +documentation. The command requires network access and is also run by the +scheduled Router Provider Drift workflow: + +```sh +pnpm --filter @comfyorg/website check:router-provider-drift +``` + ## `generate-workshop-display.ts` Imports the content team's `workshop-display.json`, keyed by catalogue model ID, From 9428b4e21e2494abbc2d970db388e6cbf69b84fd Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 06:54:20 +0000 Subject: [PATCH 26/35] test(website): prove router source comparisons Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4172055658 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4172055676 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4172055678 --- .../scripts/router-provider-drift.test.ts | 61 ++++++++++++++++--- apps/website/scripts/router-provider-drift.ts | 8 +-- 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/apps/website/scripts/router-provider-drift.test.ts b/apps/website/scripts/router-provider-drift.test.ts index 8a89383b11a..0d773a70512 100644 --- a/apps/website/scripts/router-provider-drift.test.ts +++ b/apps/website/scripts/router-provider-drift.test.ts @@ -93,7 +93,7 @@ it('reads the provider coverage table from Markdown', () => { it('detects drift with controlled published-source fixtures', async () => { const coverageRows = ROUTER_PROVIDER_COVERAGE.map( ({ docsName, docsUrl, name }) => - `| [${docsName ?? name}](${docsUrl.slice(DOCS_ORIGIN.length)}) | ✓ |` + `| [${docsName ?? name}](${new URL(docsUrl).pathname}) | ✓ |` ).join('\n') const coverage = ` ## Provider coverage @@ -102,10 +102,10 @@ it('detects drift with controlled published-source fixtures', async () => { ${coverageRows} ` const catalog = ROUTER_COMFY_ONLY_PREVIEW.map( - ({ docsUrl, name }) => `[${name}](${docsUrl.slice(DOCS_ORIGIN.length)})` + ({ docsUrl, name }) => `[${name}](${new URL(docsUrl).pathname})` ).join('\n') const fetchImpl = vi.fn(async (input) => { - const url = String(input) + const url = input instanceof Request ? input.url : String(input) if (url.endsWith('/providers.md')) return new Response(coverage) if (url.endsWith('/models.md')) return new Response(catalog) const row = ROUTER_PROVIDER_COVERAGE.find(({ modelId }) => @@ -124,10 +124,57 @@ ${coverageRows} checkRouterProviderDrift({ fetchImpl, sleep: async () => {} }) ).resolves.toBeUndefined() - fetchImpl.mockImplementationOnce(async () => - Promise.resolve(new Response(coverage.replace('Model / provider', 'Drift'))) + const [driftedRow] = ROUTER_PROVIDER_COVERAGE + const driftedProviderFetch = vi.fn(async (input) => + (input instanceof Request ? input.url : String(input)).endsWith( + `/${driftedRow.modelId}.json` + ) + ? new Response( + JSON.stringify({ + 'x-comfy-router-alt-providers': [ + ...driftedRow.providers, + 'drift-provider' + ].map((provider) => ({ provider })) + }) + ) + : fetchImpl(input) ) await expect( - checkRouterProviderDrift({ fetchImpl, sleep: async () => {} }) - ).rejects.toThrow() + checkRouterProviderDrift({ + fetchImpl: driftedProviderFetch, + sleep: async () => {} + }) + ).rejects.toThrow(/drift-provider/) + + const driftedDocsFetch = vi.fn(async (input) => { + const url = input instanceof Request ? input.url : String(input) + return url.endsWith('/providers.md') + ? new Response( + coverage.replace( + new URL(driftedRow.docsUrl).pathname, + `${new URL(driftedRow.docsUrl).pathname}-drift` + ) + ) + : fetchImpl(input) + }) + await expect( + checkRouterProviderDrift({ + fetchImpl: driftedDocsFetch, + sleep: async () => {} + }) + ).rejects.toThrow(/-drift/) + + const [previewRow] = ROUTER_COMFY_ONLY_PREVIEW + const driftedCatalogFetch = vi.fn(async (input) => { + const url = input instanceof Request ? input.url : String(input) + return url.endsWith('/models.md') + ? new Response(catalog.replace(`[${previewRow.name}]`, '[Drift]')) + : fetchImpl(input) + }) + await expect( + checkRouterProviderDrift({ + fetchImpl: driftedCatalogFetch, + sleep: async () => {} + }) + ).rejects.toThrow(new RegExp(previewRow.name)) }) diff --git a/apps/website/scripts/router-provider-drift.ts b/apps/website/scripts/router-provider-drift.ts index bad34eea1f7..2cb3f055e39 100644 --- a/apps/website/scripts/router-provider-drift.ts +++ b/apps/website/scripts/router-provider-drift.ts @@ -129,12 +129,10 @@ export async function checkRouterProviderDrift( const missingCatalogLinks = ROUTER_COMFY_ONLY_PREVIEW.filter( ({ name, docsUrl }) => - !catalogMarkdown.includes( - `[${name}](${docsUrl.slice(DOCS_ORIGIN.length)})` - ) + !catalogMarkdown.includes(`[${name}](${new URL(docsUrl).pathname})`) ).map(({ name }) => name) - const previewPathsInCoverage = ROUTER_COMFY_ONLY_PREVIEW.map(({ docsUrl }) => - docsUrl.slice(DOCS_ORIGIN.length) + const previewPathsInCoverage = ROUTER_COMFY_ONLY_PREVIEW.map( + ({ docsUrl }) => new URL(docsUrl).pathname ).filter((path) => coverageMarkdown.includes(path)) assert.deepEqual(missingCatalogLinks, []) assert.deepEqual(previewPathsInCoverage, []) From 3b1a86c0f824fd8a5ff47d4262fd42e341dc7c13 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 06:54:20 +0000 Subject: [PATCH 27/35] fix(ci): distinguish drift from workflow failures Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4172055666 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4172055670 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4172055672 --- .github/workflows/ci-router-provider-drift.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-router-provider-drift.yaml b/.github/workflows/ci-router-provider-drift.yaml index 10de354ca14..aeda9fb08c2 100644 --- a/.github/workflows/ci-router-provider-drift.yaml +++ b/.github/workflows/ci-router-provider-drift.yaml @@ -13,6 +13,7 @@ permissions: env: ISSUE_LABEL: 'area:testing' + ISSUE_MARKER: '' ISSUE_TITLE: 'Router provider coverage has drifted from published sources' concurrency: @@ -30,16 +31,18 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-frontend - name: Compare checked-in coverage with published sources + id: drift run: pnpm --filter @comfyorg/website check:router-provider-drift - name: File or update the drift issue - if: failure() + if: failure() && steps.drift.outcome == 'failure' uses: actions/github-script@v9 with: github-token: ${{ github.token }} script: | const title = process.env.ISSUE_TITLE + const marker = process.env.ISSUE_MARKER const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` - const body = `The scheduled [Router Provider Drift](${runUrl}) check failed. The checked-in Router provider coverage may no longer match the published documentation or schemas, or a published source may be unavailable. See the run for the exact failure. This issue is maintained by \`.github/workflows/ci-router-provider-drift.yaml\` and closes itself after a passing run.` + const body = `${marker}\nThe scheduled [Router Provider Drift](${runUrl}) check failed. The checked-in Router provider coverage may no longer match the published documentation or schemas, or a published source may be unavailable. See the run for the exact failure. This issue is maintained by \`.github/workflows/ci-router-provider-drift.yaml\` and closes itself after a passing run.` const openIssues = await github.paginate(github.rest.issues.listForRepo, { owner: context.repo.owner, repo: context.repo.repo, @@ -47,7 +50,9 @@ jobs: labels: process.env.ISSUE_LABEL, per_page: 100 }) - const existing = openIssues.find((issue) => issue.title === title) + const existing = openIssues.find((issue) => + !issue.pull_request && (issue.body ?? '').includes(marker) + ) if (existing) { await github.rest.issues.createComment({ owner: context.repo.owner, @@ -70,6 +75,7 @@ jobs: with: github-token: ${{ github.token }} script: | + const marker = process.env.ISSUE_MARKER const openIssues = await github.paginate(github.rest.issues.listForRepo, { owner: context.repo.owner, repo: context.repo.repo, @@ -77,7 +83,9 @@ jobs: labels: process.env.ISSUE_LABEL, per_page: 100 }) - const existing = openIssues.find((issue) => issue.title === process.env.ISSUE_TITLE) + const existing = openIssues.find((issue) => + !issue.pull_request && (issue.body ?? '').includes(marker) + ) if (!existing) return await github.rest.issues.createComment({ owner: context.repo.owner, From d8e51dd155e3ce22068904bf8c7f468fa7672000 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 06:54:20 +0000 Subject: [PATCH 28/35] docs(website): describe router drift alert Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4172055679 --- apps/website/src/config/router-providers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/website/src/config/router-providers.ts b/apps/website/src/config/router-providers.ts index 586d0764cbd..1e03a63ea4d 100644 --- a/apps/website/src/config/router-providers.ts +++ b/apps/website/src/config/router-providers.ts @@ -6,8 +6,8 @@ * model's `x-comfy-router-alt-providers`, published per model under * `router-schemas/` in the public Comfy-Org/docs repository. The rows are * copied by hand; the scheduled Router Provider Drift workflow compares them - * with the published spec. Update `ROUTER_PROVIDER_COVERAGE_VERIFIED_AT` with - * them. + * with the published spec and opens an `area:testing` issue when they drift. + * Update `ROUTER_PROVIDER_COVERAGE_VERIFIED_AT` with them. */ export const ROUTER_SERVING_PROVIDERS = [ From 4292ec2a855152f72f8db7607aa079b9076cd9b8 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 07:35:33 +0000 Subject: [PATCH 29/35] fix(website): remove stale drift test constant --- apps/website/scripts/router-provider-drift.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/website/scripts/router-provider-drift.test.ts b/apps/website/scripts/router-provider-drift.test.ts index 0d773a70512..cb09cf9707d 100644 --- a/apps/website/scripts/router-provider-drift.test.ts +++ b/apps/website/scripts/router-provider-drift.test.ts @@ -12,7 +12,6 @@ import { parseCoverageTable } from './router-provider-drift' -const DOCS_ORIGIN = 'https://docs.comfy.org' describe('fetchText', () => { it('recovers when a source fails once and then responds', async () => { From 20043b5d7927b225cb19ad3f97dd9f8f3d4edbb5 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sat, 3 Oct 2026 07:38:01 +0000 Subject: [PATCH 30/35] [automated] Apply ESLint and Oxfmt fixes --- apps/website/scripts/router-provider-drift.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/website/scripts/router-provider-drift.test.ts b/apps/website/scripts/router-provider-drift.test.ts index cb09cf9707d..993b00bc418 100644 --- a/apps/website/scripts/router-provider-drift.test.ts +++ b/apps/website/scripts/router-provider-drift.test.ts @@ -12,7 +12,6 @@ import { parseCoverageTable } from './router-provider-drift' - describe('fetchText', () => { it('recovers when a source fails once and then responds', async () => { const fetchImpl = vi From 0d64678fcbd776865d0a987116db555e7bc9739d Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 19:57:37 +0000 Subject: [PATCH 31/35] test(website): cover remaining router drift paths Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4172086268 --- .../scripts/router-provider-drift.test.ts | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/apps/website/scripts/router-provider-drift.test.ts b/apps/website/scripts/router-provider-drift.test.ts index 993b00bc418..439c8ce95e1 100644 --- a/apps/website/scripts/router-provider-drift.test.ts +++ b/apps/website/scripts/router-provider-drift.test.ts @@ -123,6 +123,25 @@ ${coverageRows} ).resolves.toBeUndefined() const [driftedRow] = ROUTER_PROVIDER_COVERAGE + const [driftedProvider] = ROUTER_SERVING_PROVIDERS + const driftedHeaderFetch = vi.fn(async (input) => { + const url = input instanceof Request ? input.url : String(input) + return url.endsWith('/providers.md') + ? new Response( + coverage.replace( + `**${driftedProvider.name}**`, + `**${driftedProvider.name}-drift**` + ) + ) + : fetchImpl(input) + }) + await expect( + checkRouterProviderDrift({ + fetchImpl: driftedHeaderFetch, + sleep: async () => {} + }) + ).rejects.toThrow(new RegExp(`${driftedProvider.name}-drift`, 'i')) + const driftedProviderFetch = vi.fn(async (input) => (input instanceof Request ? input.url : String(input)).endsWith( `/${driftedRow.modelId}.json` @@ -162,6 +181,24 @@ ${coverageRows} }) ).rejects.toThrow(/-drift/) + const driftedComfyFetch = vi.fn(async (input) => { + const url = input instanceof Request ? input.url : String(input) + return url.endsWith('/providers.md') + ? new Response( + coverage.replace( + `| [${driftedRow.docsName ?? driftedRow.name}](${new URL(driftedRow.docsUrl).pathname}) | ✓ |`, + `| [${driftedRow.docsName ?? driftedRow.name}](${new URL(driftedRow.docsUrl).pathname}) | - |` + ) + ) + : fetchImpl(input) + }) + await expect( + checkRouterProviderDrift({ + fetchImpl: driftedComfyFetch, + sleep: async () => {} + }) + ).rejects.toThrow() + const [previewRow] = ROUTER_COMFY_ONLY_PREVIEW const driftedCatalogFetch = vi.fn(async (input) => { const url = input instanceof Request ? input.url : String(input) @@ -175,4 +212,18 @@ ${coverageRows} sleep: async () => {} }) ).rejects.toThrow(new RegExp(previewRow.name)) + + const previewPath = new URL(previewRow.docsUrl).pathname + const driftedPreviewCoverageFetch = vi.fn(async (input) => { + const url = input instanceof Request ? input.url : String(input) + return url.endsWith('/providers.md') + ? new Response(`${previewPath}\n${coverage}`) + : fetchImpl(input) + }) + await expect( + checkRouterProviderDrift({ + fetchImpl: driftedPreviewCoverageFetch, + sleep: async () => {} + }) + ).rejects.toThrow(new RegExp(previewPath.replaceAll('/', '\\/'))) }) From a34f38676e510a73cd866cbdd8388e5a83d633c5 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 19:57:37 +0000 Subject: [PATCH 32/35] docs(website): clarify router drift issue behavior Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4172086271 --- apps/website/src/config/router-providers.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/website/src/config/router-providers.ts b/apps/website/src/config/router-providers.ts index 1e03a63ea4d..d91ff18f354 100644 --- a/apps/website/src/config/router-providers.ts +++ b/apps/website/src/config/router-providers.ts @@ -6,7 +6,8 @@ * model's `x-comfy-router-alt-providers`, published per model under * `router-schemas/` in the public Comfy-Org/docs repository. The rows are * copied by hand; the scheduled Router Provider Drift workflow compares them - * with the published spec and opens an `area:testing` issue when they drift. + * with the published sources and opens or updates an `area:testing` issue when + * they drift or a source is unavailable. * Update `ROUTER_PROVIDER_COVERAGE_VERIFIED_AT` with them. */ From e4bd98842fe8b5221fcc58cc8132ed0e5beb8b32 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 22:21:07 +0000 Subject: [PATCH 33/35] test(website): pin Comfy coverage drift error Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4174645893 --- apps/website/scripts/router-provider-drift.test.ts | 2 +- apps/website/scripts/router-provider-drift.ts | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/website/scripts/router-provider-drift.test.ts b/apps/website/scripts/router-provider-drift.test.ts index 439c8ce95e1..cc81399bcc7 100644 --- a/apps/website/scripts/router-provider-drift.test.ts +++ b/apps/website/scripts/router-provider-drift.test.ts @@ -197,7 +197,7 @@ ${coverageRows} fetchImpl: driftedComfyFetch, sleep: async () => {} }) - ).rejects.toThrow() + ).rejects.toThrow('every provider coverage row must be served by Comfy') const [previewRow] = ROUTER_COMFY_ONLY_PREVIEW const driftedCatalogFetch = vi.fn(async (input) => { diff --git a/apps/website/scripts/router-provider-drift.ts b/apps/website/scripts/router-provider-drift.ts index 2cb3f055e39..40691953fc3 100644 --- a/apps/website/scripts/router-provider-drift.ts +++ b/apps/website/scripts/router-provider-drift.ts @@ -116,7 +116,10 @@ export async function checkRouterProviderDrift( docs.providers, ROUTER_SERVING_PROVIDERS.map((provider) => provider.name) ) - assert.ok(docs.rows.every((row) => row.comfy === '✓')) + assert.ok( + docs.rows.every((row) => row.comfy === '✓'), + 'every provider coverage row must be served by Comfy' + ) const byDocsUrl = (a: { docsUrl: string }, b: { docsUrl: string }) => a.docsUrl.localeCompare(b.docsUrl) assert.deepEqual( From 1cbab6f39cc29c5e7e4e802f539527bd912f3a99 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 22:21:35 +0000 Subject: [PATCH 34/35] test(website): isolate router drift cases Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4174645897 --- .../scripts/router-provider-drift.test.ts | 219 +++++++++--------- 1 file changed, 106 insertions(+), 113 deletions(-) diff --git a/apps/website/scripts/router-provider-drift.test.ts b/apps/website/scripts/router-provider-drift.test.ts index cc81399bcc7..723aa9c84da 100644 --- a/apps/website/scripts/router-provider-drift.test.ts +++ b/apps/website/scripts/router-provider-drift.test.ts @@ -88,7 +88,7 @@ it('reads the provider coverage table from Markdown', () => { }) }) -it('detects drift with controlled published-source fixtures', async () => { +describe('checkRouterProviderDrift', () => { const coverageRows = ROUTER_PROVIDER_COVERAGE.map( ({ docsName, docsUrl, name }) => `| [${docsName ?? name}](${new URL(docsUrl).pathname}) | ✓ |` @@ -102,128 +102,121 @@ ${coverageRows} const catalog = ROUTER_COMFY_ONLY_PREVIEW.map( ({ docsUrl, name }) => `[${name}](${new URL(docsUrl).pathname})` ).join('\n') - const fetchImpl = vi.fn(async (input) => { - const url = input instanceof Request ? input.url : String(input) - if (url.endsWith('/providers.md')) return new Response(coverage) - if (url.endsWith('/models.md')) return new Response(catalog) - const row = ROUTER_PROVIDER_COVERAGE.find(({ modelId }) => - url.endsWith(`/${modelId}.json`) - ) - return new Response( - JSON.stringify({ - 'x-comfy-router-alt-providers': (row?.providers ?? []).map( - (provider) => ({ provider }) - ) + const [driftedRow] = ROUTER_PROVIDER_COVERAGE + const [driftedProvider] = ROUTER_SERVING_PROVIDERS + const [previewRow] = ROUTER_COMFY_ONLY_PREVIEW + + function createFetch({ + coverageMarkdown = coverage, + catalogMarkdown = catalog, + providerOverrides = {} + }: { + coverageMarkdown?: string + catalogMarkdown?: string + providerOverrides?: Readonly> + } = {}) { + return vi.fn(async (input) => { + const url = input instanceof Request ? input.url : String(input) + if (url.endsWith('/providers.md')) { + return new Response(coverageMarkdown) + } + if (url.endsWith('/models.md')) return new Response(catalogMarkdown) + const row = ROUTER_PROVIDER_COVERAGE.find(({ modelId }) => + url.endsWith(`/${modelId}.json`) + ) + const providers = row + ? (providerOverrides[row.modelId] ?? row.providers) + : [] + return new Response( + JSON.stringify({ + 'x-comfy-router-alt-providers': providers.map((provider) => ({ + provider + })) + }) + ) + }) + } + + it('accepts matching published sources', async () => { + await expect( + checkRouterProviderDrift({ + fetchImpl: createFetch(), + sleep: async () => {} }) - ) + ).resolves.toBeUndefined() }) - await expect( - checkRouterProviderDrift({ fetchImpl, sleep: async () => {} }) - ).resolves.toBeUndefined() + it('detects serving-provider header drift', async () => { + const coverageMarkdown = coverage.replace( + `**${driftedProvider.name}**`, + `**${driftedProvider.name}-drift**` + ) + await expect( + checkRouterProviderDrift({ + fetchImpl: createFetch({ coverageMarkdown }), + sleep: async () => {} + }) + ).rejects.toThrow(new RegExp(`${driftedProvider.name}-drift`, 'i')) + }) - const [driftedRow] = ROUTER_PROVIDER_COVERAGE - const [driftedProvider] = ROUTER_SERVING_PROVIDERS - const driftedHeaderFetch = vi.fn(async (input) => { - const url = input instanceof Request ? input.url : String(input) - return url.endsWith('/providers.md') - ? new Response( - coverage.replace( - `**${driftedProvider.name}**`, - `**${driftedProvider.name}-drift**` - ) - ) - : fetchImpl(input) + it('detects alternate-provider drift', async () => { + await expect( + checkRouterProviderDrift({ + fetchImpl: createFetch({ + providerOverrides: { + [driftedRow.modelId]: [...driftedRow.providers, 'drift-provider'] + } + }), + sleep: async () => {} + }) + ).rejects.toThrow(/drift-provider/) }) - await expect( - checkRouterProviderDrift({ - fetchImpl: driftedHeaderFetch, - sleep: async () => {} - }) - ).rejects.toThrow(new RegExp(`${driftedProvider.name}-drift`, 'i')) - const driftedProviderFetch = vi.fn(async (input) => - (input instanceof Request ? input.url : String(input)).endsWith( - `/${driftedRow.modelId}.json` + it('detects provider documentation row drift', async () => { + const coverageMarkdown = coverage.replace( + new URL(driftedRow.docsUrl).pathname, + `${new URL(driftedRow.docsUrl).pathname}-drift` ) - ? new Response( - JSON.stringify({ - 'x-comfy-router-alt-providers': [ - ...driftedRow.providers, - 'drift-provider' - ].map((provider) => ({ provider })) - }) - ) - : fetchImpl(input) - ) - await expect( - checkRouterProviderDrift({ - fetchImpl: driftedProviderFetch, - sleep: async () => {} - }) - ).rejects.toThrow(/drift-provider/) - - const driftedDocsFetch = vi.fn(async (input) => { - const url = input instanceof Request ? input.url : String(input) - return url.endsWith('/providers.md') - ? new Response( - coverage.replace( - new URL(driftedRow.docsUrl).pathname, - `${new URL(driftedRow.docsUrl).pathname}-drift` - ) - ) - : fetchImpl(input) + await expect( + checkRouterProviderDrift({ + fetchImpl: createFetch({ coverageMarkdown }), + sleep: async () => {} + }) + ).rejects.toThrow(/-drift/) }) - await expect( - checkRouterProviderDrift({ - fetchImpl: driftedDocsFetch, - sleep: async () => {} - }) - ).rejects.toThrow(/-drift/) - - const driftedComfyFetch = vi.fn(async (input) => { - const url = input instanceof Request ? input.url : String(input) - return url.endsWith('/providers.md') - ? new Response( - coverage.replace( - `| [${driftedRow.docsName ?? driftedRow.name}](${new URL(driftedRow.docsUrl).pathname}) | ✓ |`, - `| [${driftedRow.docsName ?? driftedRow.name}](${new URL(driftedRow.docsUrl).pathname}) | - |` - ) - ) - : fetchImpl(input) + + it('detects a provider row not served by Comfy', async () => { + const coverageMarkdown = coverage.replace( + `| [${driftedRow.docsName ?? driftedRow.name}](${new URL(driftedRow.docsUrl).pathname}) | ✓ |`, + `| [${driftedRow.docsName ?? driftedRow.name}](${new URL(driftedRow.docsUrl).pathname}) | - |` + ) + await expect( + checkRouterProviderDrift({ + fetchImpl: createFetch({ coverageMarkdown }), + sleep: async () => {} + }) + ).rejects.toThrow('every provider coverage row must be served by Comfy') }) - await expect( - checkRouterProviderDrift({ - fetchImpl: driftedComfyFetch, - sleep: async () => {} - }) - ).rejects.toThrow('every provider coverage row must be served by Comfy') - const [previewRow] = ROUTER_COMFY_ONLY_PREVIEW - const driftedCatalogFetch = vi.fn(async (input) => { - const url = input instanceof Request ? input.url : String(input) - return url.endsWith('/models.md') - ? new Response(catalog.replace(`[${previewRow.name}]`, '[Drift]')) - : fetchImpl(input) + it('detects a missing catalog link', async () => { + const catalogMarkdown = catalog.replace(`[${previewRow.name}]`, '[Drift]') + await expect( + checkRouterProviderDrift({ + fetchImpl: createFetch({ catalogMarkdown }), + sleep: async () => {} + }) + ).rejects.toThrow(new RegExp(previewRow.name)) }) - await expect( - checkRouterProviderDrift({ - fetchImpl: driftedCatalogFetch, - sleep: async () => {} - }) - ).rejects.toThrow(new RegExp(previewRow.name)) - - const previewPath = new URL(previewRow.docsUrl).pathname - const driftedPreviewCoverageFetch = vi.fn(async (input) => { - const url = input instanceof Request ? input.url : String(input) - return url.endsWith('/providers.md') - ? new Response(`${previewPath}\n${coverage}`) - : fetchImpl(input) + + it('detects a preview path in provider coverage', async () => { + const previewPath = new URL(previewRow.docsUrl).pathname + await expect( + checkRouterProviderDrift({ + fetchImpl: createFetch({ + coverageMarkdown: `${previewPath}\n${coverage}` + }), + sleep: async () => {} + }) + ).rejects.toThrow(new RegExp(previewPath.replaceAll('/', '\\/'))) }) - await expect( - checkRouterProviderDrift({ - fetchImpl: driftedPreviewCoverageFetch, - sleep: async () => {} - }) - ).rejects.toThrow(new RegExp(previewPath.replaceAll('/', '\\/'))) }) From 62865c2d810a0870aca5d5895edd55763419f2a7 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 3 Oct 2026 22:21:42 +0000 Subject: [PATCH 35/35] test(website): match drift errors as text Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19713#discussion_r4174645902 --- apps/website/scripts/router-provider-drift.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/website/scripts/router-provider-drift.test.ts b/apps/website/scripts/router-provider-drift.test.ts index 723aa9c84da..145e58ab6db 100644 --- a/apps/website/scripts/router-provider-drift.test.ts +++ b/apps/website/scripts/router-provider-drift.test.ts @@ -156,7 +156,7 @@ ${coverageRows} fetchImpl: createFetch({ coverageMarkdown }), sleep: async () => {} }) - ).rejects.toThrow(new RegExp(`${driftedProvider.name}-drift`, 'i')) + ).rejects.toThrow(`${driftedProvider.name}-drift`) }) it('detects alternate-provider drift', async () => { @@ -205,7 +205,7 @@ ${coverageRows} fetchImpl: createFetch({ catalogMarkdown }), sleep: async () => {} }) - ).rejects.toThrow(new RegExp(previewRow.name)) + ).rejects.toThrow(previewRow.name) }) it('detects a preview path in provider coverage', async () => { @@ -217,6 +217,6 @@ ${coverageRows} }), sleep: async () => {} }) - ).rejects.toThrow(new RegExp(previewPath.replaceAll('/', '\\/'))) + ).rejects.toThrow(previewPath) }) })