From abd800f589dd4374f1e0e0f9aedaf9631cdb62a7 Mon Sep 17 00:00:00 2001 From: Jonathan Hefner Date: Wed, 23 Sep 2026 15:28:27 -0700 Subject: [PATCH] Check exclusion of the legacy HTTP transport type Clients that also support legacy MCP formats may accept `type: "http"` through a shared parser even when they require an explicit transport type. Agent Plugins requires `streamable-http`, so missing-type coverage does not catch this compatibility error. Add one otherwise valid Recovery entry using the existing HTTP helper and invalid-entry evaluator. Require complete exclusion evidence with working Recovery and HTTP controls; treat advertisement or attributable runtime evidence as failure and ambiguous attempts as not verified. Cover the isolated schema defect, permissive reference-client execution, canonical reporting, and native Codex rejection without adding setup steps. --- .../dist/serve-http.mjs | 1 + .../src/serve-http.mjs | 1 + .../README.md | 1 + .../mcp.json | 4 +++ .../skills/run-conformance/SKILL.md | 4 +-- .../agent-plugins-conformance/src/cases.mjs | 1 + .../agent-plugins-conformance/src/report.mjs | 1 + scripts/smoke-codex.mjs | 2 +- test/http-server.test.mjs | 34 +++++++++++++++++++ test/package.test.mjs | 20 +++++++++++ test/probe.test.mjs | 2 +- test/record-report.test.mjs | 1 + test/report-human.test.mjs | 11 +++--- test/report.test.mjs | 19 ++++++----- 14 files changed, 84 insertions(+), 18 deletions(-) diff --git a/plugins/agent-plugins-conformance-core/dist/serve-http.mjs b/plugins/agent-plugins-conformance-core/dist/serve-http.mjs index fe50616..852e47d 100644 --- a/plugins/agent-plugins-conformance-core/dist/serve-http.mjs +++ b/plugins/agent-plugins-conformance-core/dist/serve-http.mjs @@ -24366,6 +24366,7 @@ var pathname = "/conformance/mcp"; var url = `http://${host}:${port}${pathname}?value=$APC_HTTP_VALUE`; var recoveryRoutes = /* @__PURE__ */ new Map([ ["/conformance/recovery-http-relative-url", "recovery-http-relative-url"], + ["/conformance/recovery-http-type", "recovery-http-type"], ["/conformance/recovery-http-header-name", "recovery-http-header-name"], ["/conformance/recovery-http-header-value", "recovery-http-header-value"], ["/conformance/recovery-http-userinfo", "recovery-http-userinfo"], diff --git a/plugins/agent-plugins-conformance-core/src/serve-http.mjs b/plugins/agent-plugins-conformance-core/src/serve-http.mjs index 2c95e04..72fd620 100644 --- a/plugins/agent-plugins-conformance-core/src/serve-http.mjs +++ b/plugins/agent-plugins-conformance-core/src/serve-http.mjs @@ -12,6 +12,7 @@ const pathname = '/conformance/mcp'; const url = `http://${host}:${port}${pathname}?value=$APC_HTTP_VALUE`; const recoveryRoutes = new Map([ ['/conformance/recovery-http-relative-url', 'recovery-http-relative-url'], + ['/conformance/recovery-http-type', 'recovery-http-type'], ['/conformance/recovery-http-header-name', 'recovery-http-header-name'], ['/conformance/recovery-http-header-value', 'recovery-http-header-value'], ['/conformance/recovery-http-userinfo', 'recovery-http-userinfo'], diff --git a/plugins/agent-plugins-conformance-recovery/README.md b/plugins/agent-plugins-conformance-recovery/README.md index 4d76d9e..2bbdfcc 100644 --- a/plugins/agent-plugins-conformance-recovery/README.md +++ b/plugins/agent-plugins-conformance-recovery/README.md @@ -14,6 +14,7 @@ Install it alongside the other suite plugins and follow the [run instructions](. | `filesystem.containment.cwd-symlink-escape` | A server whose working directory escapes the plugin root through a symlink is excluded. | | `mcp.stdio.config.unknown-field` | A server with an unknown configuration field is excluded. | | `mcp.config.missing-type` | A server without an explicit transport type is excluded. | +| `mcp.config.legacy-http-type` | A server using the legacy `http` transport type is excluded. | | `mcp.stdio.env.reserved-plugin-root` | A server that configures `PLUGIN_ROOT` in its `env` is excluded. | | `mcp.stdio.env.reserved-plugin-data` | A server that configures `PLUGIN_DATA` in its `env` is excluded. | | `mcp.streamable-http.url.relative` | An HTTP server with a scheme-relative URL (`//host/path`) is excluded. | diff --git a/plugins/agent-plugins-conformance-recovery/mcp.json b/plugins/agent-plugins-conformance-recovery/mcp.json index 976ce06..104714b 100644 --- a/plugins/agent-plugins-conformance-recovery/mcp.json +++ b/plugins/agent-plugins-conformance-recovery/mcp.json @@ -59,6 +59,10 @@ "type": "streamable-http", "url": "//127.0.0.1:43187/conformance/recovery-http-relative-url" }, + "recovery-http-type": { + "type": "http", + "url": "http://127.0.0.1:43187/conformance/recovery-http-type" + }, "recovery-http-header-name": { "type": "streamable-http", "url": "http://127.0.0.1:43187/conformance/recovery-http-header-name", diff --git a/plugins/agent-plugins-conformance/skills/run-conformance/SKILL.md b/plugins/agent-plugins-conformance/skills/run-conformance/SKILL.md index cd380a6..0890c33 100644 --- a/plugins/agent-plugins-conformance/skills/run-conformance/SKILL.md +++ b/plugins/agent-plugins-conformance/skills/run-conformance/SKILL.md @@ -60,7 +60,7 @@ Collect all fixture skills and MCP tools through the client's normal mechanisms. } ``` -3. Inspect the client's MCP tool catalog for the `observe` tool of each invalid Recovery server with a probe (`recovery-cwd-invalid-form`, `recovery-cwd-escape`, `recovery-cwd-data-escape`, `recovery-cwd-symlink-escape`, `recovery-unknown-field`, `recovery-missing-type`, `recovery-env-plugin-root`, `recovery-env-plugin-data`, `recovery-http-relative-url`, `recovery-http-fragment`, `recovery-http-userinfo`, `recovery-http-duplicate-headers`, `recovery-http-header-name`, `recovery-http-header-value`, `recovery-sse-relative-url`, `recovery-sse-fragment`, `recovery-sse-userinfo`, `recovery-sse-duplicate-headers`, `recovery-sse-header-name`, and `recovery-sse-header-value`). Each description identifies the fixture server ID; names may be client-namespaced. Set `advertised` to `true` if the tool is advertised, even in a partial catalog, and record that before calling it. Set `advertised` to `false` only after a known attempt to load Recovery and a usable complete client inventory that excludes the tool, including any deferred tools. For `recovery-cwd-symlink-escape`, a completed startup failure still permits `advertised: false` when that complete inventory shows no tools for the server; installation may have removed its working-directory symlink. For the other servers, leave the observation missing if startup or tool discovery fails. For every server, leave it missing if the load scope or inventory completeness is unknown, unless the tool is advertised. A failed call or a guessed tool name does not establish absence. No rejection diagnostic is required. Record each server separately through the same reporter, using its fixture server ID: +3. Inspect the client's MCP tool catalog for the `observe` tool of each invalid Recovery server with a probe (`recovery-cwd-invalid-form`, `recovery-cwd-escape`, `recovery-cwd-data-escape`, `recovery-cwd-symlink-escape`, `recovery-unknown-field`, `recovery-missing-type`, `recovery-env-plugin-root`, `recovery-env-plugin-data`, `recovery-http-type`, `recovery-http-relative-url`, `recovery-http-fragment`, `recovery-http-userinfo`, `recovery-http-duplicate-headers`, `recovery-http-header-name`, `recovery-http-header-value`, `recovery-sse-relative-url`, `recovery-sse-fragment`, `recovery-sse-userinfo`, `recovery-sse-duplicate-headers`, `recovery-sse-header-name`, and `recovery-sse-header-value`). Each description identifies the fixture server ID; names may be client-namespaced. Set `advertised` to `true` if the tool is advertised, even in a partial catalog, and record that before calling it. Set `advertised` to `false` only after a known attempt to load Recovery and a usable complete client inventory that excludes the tool, including any deferred tools. For `recovery-cwd-symlink-escape`, a completed startup failure still permits `advertised: false` when that complete inventory shows no tools for the server; installation may have removed its working-directory symlink. For the other servers, leave the observation missing if startup or tool discovery fails. For every server, leave it missing if the load scope or inventory completeness is unknown, unless the tool is advertised. A failed call or a guessed tool name does not establish absence. No rejection diagnostic is required. Record each server separately through the same reporter, using its fixture server ID: ```json { @@ -69,7 +69,7 @@ Collect all fixture skills and MCP tools through the client's normal mechanisms. } ``` -4. Find the `observe` tools on the Core fixture servers (`default`, `relative`, `root`, `data`, `http`, `http-redirect`, `sse`, `sse-header-precedence`, `sse-redirect`, and `sse-endpoint-origin`) and the Recovery fixture servers (`recovery-valid`, `recovery-cwd-invalid-form`, `recovery-cwd-escape`, `recovery-cwd-data-escape`, `recovery-cwd-symlink-escape`, `recovery-unknown-field`, `recovery-missing-type`, `recovery-env-plugin-root`, `recovery-env-plugin-data`, `recovery-http-relative-url`, `recovery-http-fragment`, `recovery-http-userinfo`, `recovery-http-duplicate-headers`, `recovery-http-header-name`, `recovery-http-header-value`, `recovery-sse-relative-url`, `recovery-sse-fragment`, `recovery-sse-userinfo`, `recovery-sse-duplicate-headers`, `recovery-sse-header-name`, and `recovery-sse-header-value`) through the client's normal MCP mechanism. Tool names may be client-namespaced. The remote MCP tool descriptions identify their fixture server IDs; retain that association when calling each tool so that an error can be recorded for the same server. Call each available tool with `{}`. First inspect whether the client returned a successful MCP result or a native discovery/call error. For a native remote MCP error, follow step 5 without parsing it as observation JSON. For a native stdio error, leave that server’s runtime observation missing; retain any discovery observation already recorded. For a successful result, take the observation from `structuredContent` (some clients show `structured_content`), or parse the JSON in the tool's text content. Immediately record that complete object; exclude the MCP result wrapper containing `content` or `structuredContent`. Preserve the returned paths and values exactly. The `http-redirect` and `sse-redirect` fixtures redirect requests to another local origin, and `sse-endpoint-origin` advertises a message endpoint on that origin. Do not authorize forwarding configured headers to these destinations. +4. Find the `observe` tools on the Core fixture servers (`default`, `relative`, `root`, `data`, `http`, `http-redirect`, `sse`, `sse-header-precedence`, `sse-redirect`, and `sse-endpoint-origin`) and the Recovery fixture servers (`recovery-valid`, `recovery-cwd-invalid-form`, `recovery-cwd-escape`, `recovery-cwd-data-escape`, `recovery-cwd-symlink-escape`, `recovery-unknown-field`, `recovery-missing-type`, `recovery-env-plugin-root`, `recovery-env-plugin-data`, `recovery-http-type`, `recovery-http-relative-url`, `recovery-http-fragment`, `recovery-http-userinfo`, `recovery-http-duplicate-headers`, `recovery-http-header-name`, `recovery-http-header-value`, `recovery-sse-relative-url`, `recovery-sse-fragment`, `recovery-sse-userinfo`, `recovery-sse-duplicate-headers`, `recovery-sse-header-name`, and `recovery-sse-header-value`) through the client's normal MCP mechanism. Tool names may be client-namespaced. The remote MCP tool descriptions identify their fixture server IDs; retain that association when calling each tool so that an error can be recorded for the same server. Call each available tool with `{}`. First inspect whether the client returned a successful MCP result or a native discovery/call error. For a native remote MCP error, follow step 5 without parsing it as observation JSON. For a native stdio error, leave that server’s runtime observation missing; retain any discovery observation already recorded. For a successful result, take the observation from `structuredContent` (some clients show `structured_content`), or parse the JSON in the tool's text content. Immediately record that complete object; exclude the MCP result wrapper containing `content` or `structuredContent`. Preserve the returned paths and values exactly. The `http-redirect` and `sse-redirect` fixtures redirect requests to another local origin, and `sse-endpoint-origin` advertises a message endpoint on that origin. Do not authorize forwarding configured headers to these destinations. 5. If a completed remote MCP discovery or call attempt instead produces a native error, record it as error evidence: ```json diff --git a/plugins/agent-plugins-conformance/src/cases.mjs b/plugins/agent-plugins-conformance/src/cases.mjs index d98db56..00eb183 100644 --- a/plugins/agent-plugins-conformance/src/cases.mjs +++ b/plugins/agent-plugins-conformance/src/cases.mjs @@ -17,6 +17,7 @@ export const CASES = Object.freeze([ ['filesystem.containment.cwd-symlink-escape', 'Working-directory symlink containment', ['4.1', '7.2.1', '7.2.2']], ['mcp.stdio.config.unknown-field', 'Unknown MCP server field rejection', ['7.2.1', '7.2.2']], ['mcp.config.missing-type', 'Missing MCP transport type exclusion', ['7.2.1', '7.2.2']], + ['mcp.config.legacy-http-type', 'Legacy HTTP transport type exclusion', ['7.2.1', '7.2.2']], ['mcp.stdio.env.reserved-plugin-root', 'Configured PLUGIN_ROOT exclusion', ['7.2.2', '9.2']], ['mcp.stdio.env.reserved-plugin-data', 'Configured PLUGIN_DATA exclusion', ['7.2.2', '9.2']], ['mcp.stdio.env.plugin-root', 'Plugin root environment', ['9.1']], diff --git a/plugins/agent-plugins-conformance/src/report.mjs b/plugins/agent-plugins-conformance/src/report.mjs index b9c9262..5881c16 100644 --- a/plugins/agent-plugins-conformance/src/report.mjs +++ b/plugins/agent-plugins-conformance/src/report.mjs @@ -14,6 +14,7 @@ const INVALID_STDIO_SERVERS = Object.freeze({ 'recovery-env-plugin-data': 'mcp.stdio.env.reserved-plugin-data', }); const INVALID_HTTP_SERVERS = Object.freeze({ + 'recovery-http-type': 'mcp.config.legacy-http-type', 'recovery-http-relative-url': 'mcp.streamable-http.url.relative', 'recovery-http-fragment': 'mcp.streamable-http.url.fragment', 'recovery-http-userinfo': 'mcp.streamable-http.url.userinfo', diff --git a/scripts/smoke-codex.mjs b/scripts/smoke-codex.mjs index 6c7af78..e147064 100644 --- a/scripts/smoke-codex.mjs +++ b/scripts/smoke-codex.mjs @@ -204,7 +204,7 @@ try { for (const server of ['recovery-cwd-invalid-form', 'recovery-cwd-escape', 'recovery-cwd-data-escape', 'recovery-cwd-symlink-escape', 'recovery-unknown-field', 'recovery-missing-type', 'recovery-env-plugin-root', 'recovery-env-plugin-data', 'recovery-http-relative-url', 'recovery-http-fragment', 'recovery-http-userinfo', 'recovery-http-duplicate-headers', - 'recovery-http-header-name', 'recovery-http-header-value', ...invalidSseServers]) { + 'recovery-http-header-name', 'recovery-http-header-value', 'recovery-http-type', ...invalidSseServers]) { const entry = status.data.find(({ name }) => name === server); const advertised = entry !== undefined && Object.hasOwn(entry.tools, 'observe'); const symlinkStartupFailure = server === 'recovery-cwd-symlink-escape' && diff --git a/test/http-server.test.mjs b/test/http-server.test.mjs index 755da0d..dc7fbe1 100644 --- a/test/http-server.test.mjs +++ b/test/http-server.test.mjs @@ -650,6 +650,40 @@ test('standalone HTTP bundle serves independent MCP requests and stops cleanly', } }); + await t.test('legacy HTTP type route has a valid control and a deliberately permissive failure witness', async (t) => { + const mcp = JSON.parse(await readFile(new URL('../plugins/agent-plugins-conformance-recovery/mcp.json', import.meta.url), 'utf8')); + const configured = mcp.mcpServers['recovery-http-type']; + assert.deepEqual(configured, { + type: 'http', + url: 'http://127.0.0.1:43187/conformance/recovery-http-type', + }); + const normalizedForHarness = { ...configured, type: 'streamable-http' }; + const client = new Client({ name: 'legacy-http-type-reference-test', version: '1' }); + t.after(() => client.close()); + await client.connect(new StreamableHTTPClientTransport(new URL(normalizedForHarness.url))); + assert.equal(client.getServerVersion().name, 'agent-plugins-conformance-recovery-http-type'); + const tools = (await client.listTools()).tools; + assert.deepEqual(tools.map(({ name }) => name), ['observe']); + assert.match(tools[0].description, /server ID: recovery-http-type\./); + const result = await client.callTool({ name: 'observe', arguments: {} }); + const observation = { + kind: 'mcp-streamable-http', server: 'recovery-http-type', serverHealthCheck: 'passed', + evidence: { + type: 'request', version: 1, pathname: '/conformance/recovery-http-type', + query: [], headers: { 'x-apc-fixture': null }, + }, + }; + assert.equal(result.isError, undefined); + assert.deepEqual(result.structuredContent, { + kind: observation.kind, server: observation.server, evidence: observation.evidence, + }); + assert.deepEqual(JSON.parse(result.content[0].text), result.structuredContent); + const report = buildReport({ schemaVersion: 1, observations: [observation] }); + assert.equal(report.results.find(({ id }) => id === 'mcp.config.legacy-http-type').status, 'fail'); + assert.deepEqual(report.observations, [observation]); + assert.equal(configured.type, 'http'); + }); + await t.test('userinfo route has a valid control and a deliberately permissive failure witness', async (t) => { const mcp = JSON.parse(await readFile(new URL('../plugins/agent-plugins-conformance-recovery/mcp.json', import.meta.url), 'utf8')); const configured = mcp.mcpServers['recovery-http-userinfo']; diff --git a/test/package.test.mjs b/test/package.test.mjs index 1c22072..5a3233b 100644 --- a/test/package.test.mjs +++ b/test/package.test.mjs @@ -128,6 +128,24 @@ test('recovery fixture contains exactly the intended invalid manifest and MCP fi type: 'streamable-http', url: '//127.0.0.1:43187/conformance/recovery-http-relative-url', }); + const legacyHttpType = mcp.mcpServers['recovery-http-type']; + assert.deepEqual(legacyHttpType, { + type: 'http', + url: 'http://127.0.0.1:43187/conformance/recovery-http-type', + }); + const isolatedLegacyHttpType = { + $schema: mcp.$schema, + mcpServers: { 'recovery-http-type': structuredClone(legacyHttpType) }, + }; + assert.equal(validateMcp(isolatedLegacyHttpType), false); + isolatedLegacyHttpType.mcpServers['recovery-http-type'].type = 'streamable-http'; + const { type: correctedHttpType, ...correctedHttpRemainder } = + isolatedLegacyHttpType.mcpServers['recovery-http-type']; + const { type: legacyType, ...legacyHttpRemainder } = legacyHttpType; + assert.equal(legacyType, 'http'); + assert.equal(correctedHttpType, 'streamable-http'); + assert.deepEqual(correctedHttpRemainder, legacyHttpRemainder); + assert.equal(validateMcp(isolatedLegacyHttpType), true, JSON.stringify(validateMcp.errors)); assert.deepEqual(mcp.mcpServers['recovery-http-header-name'], { type: 'streamable-http', url: 'http://127.0.0.1:43187/conformance/recovery-http-header-name', @@ -199,6 +217,7 @@ test('recovery fixture contains exactly the intended invalid manifest and MCP fi ['/mcpServers/recovery-invalid', '/mcpServers/recovery-cwd-invalid-form', '/mcpServers/recovery-unknown-field', '/mcpServers/recovery-missing-type', '/mcpServers/recovery-http-relative-url', + '/mcpServers/recovery-http-type', '/mcpServers/recovery-env-plugin-root', '/mcpServers/recovery-env-plugin-data'] .some((prefix) => instancePath.startsWith(prefix))), JSON.stringify(validateMcp.errors)); @@ -232,6 +251,7 @@ test('recovery fixture contains exactly the intended invalid manifest and MCP fi schemaValidMcp.mcpServers['recovery-cwd-invalid-form'].cwd = './'; schemaValidMcp.mcpServers['recovery-http-relative-url'].url = `http:${schemaValidMcp.mcpServers['recovery-http-relative-url'].url}`; + schemaValidMcp.mcpServers['recovery-http-type'].type = 'streamable-http'; assert.equal(validateMcp(schemaValidMcp), true, JSON.stringify(validateMcp.errors)); }); diff --git a/test/probe.test.mjs b/test/probe.test.mjs index e2848a0..19f70db 100644 --- a/test/probe.test.mjs +++ b/test/probe.test.mjs @@ -94,7 +94,7 @@ test('copied plugin runs only MCP observation tools without node_modules', { tim const direct = buildReport(reportInput); assert.equal(direct.summary.fail, 0); assert.equal(direct.summary.pass, 15); - assert.equal(direct.summary.not_verified, 36); + assert.equal(direct.summary.not_verified, 37); }); test('copied recovery plugin serves exact valid and invalid-server observations without runtime dependencies', { timeout: 30_000 }, async (t) => { diff --git a/test/record-report.test.mjs b/test/record-report.test.mjs index c53318c..dc38c82 100644 --- a/test/record-report.test.mjs +++ b/test/record-report.test.mjs @@ -48,6 +48,7 @@ const invalidServerCases = [ ['recovery-env-plugin-data', 'mcp.stdio.env.reserved-plugin-data'], ]; const invalidHttpServerCases = [ + ['recovery-http-type', 'mcp.config.legacy-http-type'], ['recovery-http-relative-url', 'mcp.streamable-http.url.relative'], ['recovery-http-fragment', 'mcp.streamable-http.url.fragment'], ['recovery-http-duplicate-headers', 'mcp.streamable-http.headers.duplicate-names'], diff --git a/test/report-human.test.mjs b/test/report-human.test.mjs index c04886b..53acefb 100644 --- a/test/report-human.test.mjs +++ b/test/report-human.test.mjs @@ -52,9 +52,9 @@ test('complete stdio and recovery evidence leaves optional HTTP and SSE checks u '', 'Checks Passed Failed Not verified', 'Skills 3 0 0', - 'MCP 21 0 23', + 'MCP 21 0 24', 'Filesystem 4 0 0', - 'Total 28 0 23', + 'Total 28 0 24', ].join('\n')); const missing = human.split('\n\nNot verified\n')[1]; for (const id of [ @@ -62,6 +62,7 @@ test('complete stdio and recovery evidence leaves optional HTTP and SSE checks u 'mcp.streamable-http.url.literal-route-and-query', 'mcp.streamable-http.headers.cross-origin-redirect', 'mcp.streamable-http.headers.literal-value', + 'mcp.config.legacy-http-type', 'mcp.streamable-http.url.relative', 'mcp.streamable-http.url.fragment', 'mcp.streamable-http.headers.duplicate-names', @@ -90,7 +91,7 @@ test('all-unverified human report preserves every saved missing-evidence result' value.observations = []; const human = formatReport(buildReport(value)); assert.match(human, /Skills\s+0\s+0\s+3/); - assert.match(human, /MCP\s+0\s+0\s+44/); + assert.match(human, /MCP\s+0\s+0\s+45/); assert.match(human, /Filesystem\s+0\s+0\s+4/); assert.match(human, /Missing skill observations: conformance-alpha, conformance-beta\./); for (const result of buildReport(value).results) assert.ok(human.includes(`(${result.id})`)); @@ -102,7 +103,7 @@ test('mixed human report puts failures before missing-evidence details', () => { value.observations = [value.observations[2]]; value.observations[0].evidence.env.APC_VALUE = 'incorrect configured value'; const human = formatReport(buildReport(value)); - assert.match(human, /MCP\s+6\s+1\s+37/); + assert.match(human, /MCP\s+6\s+1\s+38/); assert.match(human, /Filesystem\s+1\s+0\s+3/); assert.match(human, /Configured environment \(mcp.stdio.env.configured-value\)/); assert.match(human, /"fixture value with spaces"/); @@ -157,7 +158,7 @@ test('human report renders saved warnings independently of result status', () => const writable = value.results.find(({ id }) => id === 'filesystem.data.writable'); writable.status = 'not_verified'; writable.warning = 'Saved cleanup warning for /state/plugin/leftover.'; - value.summary = { pass: 15, fail: 0, not_verified: 36, total: 51 }; + value.summary = { pass: 15, fail: 0, not_verified: 37, total: 52 }; value.observations = []; const human = formatReport(value); diff --git a/test/report.test.mjs b/test/report.test.mjs index 05eced8..797585c 100644 --- a/test/report.test.mjs +++ b/test/report.test.mjs @@ -51,6 +51,7 @@ const invalidServerCases = [ ['recovery-env-plugin-data', 'mcp.stdio.env.reserved-plugin-data'], ]; const invalidHttpServerCases = [ + ['recovery-http-type', 'mcp.config.legacy-http-type'], ['recovery-http-relative-url', 'mcp.streamable-http.url.relative'], ['recovery-http-fragment', 'mcp.streamable-http.url.fragment'], ['recovery-http-duplicate-headers', 'mcp.streamable-http.headers.duplicate-names'], @@ -70,7 +71,7 @@ const invalidSseServerCases = [ test('complete stdio and skill core evidence passes its cases and preserves canonical evidence', () => { const value = input(); const report = buildReport(value); - assert.deepEqual(report.summary, { pass: 16, fail: 0, not_verified: 35, total: 51 }); + assert.deepEqual(report.summary, { pass: 16, fail: 0, not_verified: 36, total: 52 }); assert.deepEqual(report.results.map(({ id }) => id), CASE_IDS); assert.equal(report.specVersion, '1.0.0'); assert.deepEqual(result(report, 'mcp.stdio.env.plugin-root').specSections, ['9.1']); @@ -83,19 +84,19 @@ test('complete stdio and skill core evidence passes its cases and preserves cano test('recovery witnesses extend the canonical report without changing core-only results', () => { const coreOnly = buildReport(input()); - assert.deepEqual(coreOnly.summary, { pass: 16, fail: 0, not_verified: 35, total: 51 }); + assert.deepEqual(coreOnly.summary, { pass: 16, fail: 0, not_verified: 36, total: 52 }); assert.equal(result(coreOnly, 'skills.recovery.valid-skill-available').status, 'not_verified'); assert.equal(result(coreOnly, 'mcp.stdio.recovery.valid-server-available').status, 'not_verified'); const recoveryOnly = buildReport({ schemaVersion: 1, observations: recoveryObservations() }); - assert.deepEqual(recoveryOnly.summary, { pass: 2, fail: 0, not_verified: 49, total: 51 }); + assert.deepEqual(recoveryOnly.summary, { pass: 2, fail: 0, not_verified: 50, total: 52 }); assert.equal(result(recoveryOnly, 'skills.recovery.valid-skill-available').status, 'pass'); assert.equal(result(recoveryOnly, 'mcp.stdio.recovery.valid-server-available').status, 'pass'); const combinedInput = input(); combinedInput.observations.push(...recoveryObservations()); const combined = buildReport(combinedInput); - assert.deepEqual(combined.summary, { pass: 19, fail: 0, not_verified: 32, total: 51 }); + assert.deepEqual(combined.summary, { pass: 19, fail: 0, not_verified: 33, total: 52 }); assert.deepEqual(combined.results.map(({ id }) => id), CASE_IDS); const recoveryIds = new Set([ 'skills.recovery.valid-skill-available', @@ -118,7 +119,7 @@ test('recovery witnesses extend the canonical report without changing core-only assert.equal(JSON.stringify(buildReport(reordered)), JSON.stringify(combined)); const missing = buildReport({ schemaVersion: 1, observations: [] }); - assert.deepEqual(missing.summary, { pass: 0, fail: 0, not_verified: 51, total: 51 }); + assert.deepEqual(missing.summary, { pass: 0, fail: 0, not_verified: 52, total: 52 }); }); test('an incorrect recovery skill marker fails its availability check independently', () => { @@ -129,7 +130,7 @@ test('an incorrect recovery skill marker fails its availability check independen assert.equal(availability.status, 'fail'); assert.match(availability.detail, /expected "APC_RECOVERY_VALID_V1"; observed "wrong recovery marker"/); assert.equal(result(report, 'mcp.stdio.recovery.valid-server-available').status, 'pass'); - assert.deepEqual(report.summary, { pass: 1, fail: 1, not_verified: 49, total: 51 }); + assert.deepEqual(report.summary, { pass: 1, fail: 1, not_verified: 50, total: 52 }); }); test('report bytes are deterministic across observation and property orders', () => { @@ -144,11 +145,11 @@ test('missing observations remain unverified and every result identifies its hie const value = input(); value.observations = []; const report = buildReport(value); - assert.deepEqual(report.summary, { pass: 0, fail: 0, not_verified: 51, total: 51 }); + assert.deepEqual(report.summary, { pass: 0, fail: 0, not_verified: 52, total: 52 }); const skills = report.results.filter(({ id }) => id.startsWith('skills.')); const mcp = report.results.filter(({ id }) => id.startsWith('mcp.')); assert.deepEqual(skills.map(({ id }) => id), ['skills.discovery.immediate-children', 'skills.recovery.valid-skill-available', 'skills.recovery.invalid-mcp-document']); - assert.equal(mcp.length, 44); + assert.equal(mcp.length, 45); assert.equal(report.results.filter(({ id }) => id.startsWith('filesystem.')).length, 4); assert.ok(mcp.some(({ id }) => id === 'mcp.stdio.env.plugin-root')); for (const result of report.results) { @@ -564,7 +565,7 @@ test('malformed MCP skill evidence affects only its own result for missing, corr assert.deepEqual(result(report, id).specSections, ['7.2.2']); assert.deepEqual(report.results.filter((item) => item.id !== id), baseline.results.filter((item) => item.id !== id)); assert.deepEqual(report.observations.find(({ skill }) => skill === observation.skill), observation); - assert.deepEqual(report.summary, { pass: status === 'pass' ? 20 : 19, fail: status === 'fail' ? 1 : 0, not_verified: 31, total: 51 }); + assert.deepEqual(report.summary, { pass: status === 'pass' ? 20 : 19, fail: status === 'fail' ? 1 : 0, not_verified: 32, total: 52 }); if (status === 'fail') assert.match(result(report, id).detail, /expected "APC_INVALID_MCP_VALID_V1"; observed "incorrect marker"/); const reversed = { schemaVersion: 1, observations: [...value.observations, observation].reverse() }; assert.equal(JSON.stringify(buildReport(reversed)), JSON.stringify(report));