diff --git a/plugins/agent-plugins-conformance-core/README.md b/plugins/agent-plugins-conformance-core/README.md index 9164745..cf883c1 100644 --- a/plugins/agent-plugins-conformance-core/README.md +++ b/plugins/agent-plugins-conformance-core/README.md @@ -6,6 +6,8 @@ Install this fixture alongside [Agent Plugins Conformance](../agent-plugins-conf Node.js 22 or newer must be available as `node` on the client's executable search path. The committed MCP bundle includes the official MCP SDK and requires no dependency installation or build. +The command-token check includes POSIX and Windows servers. Only the server for the current platform is expected to run; the other may produce a startup error. + ## Optional HTTP setup To include checks for MCP servers using Streamable HTTP or legacy HTTP+SSE, start the bundled server in a separate terminal **before the client loads the plugin**: @@ -28,9 +30,10 @@ Replace the path with a downloaded or installed copy of this plugin; the command | `mcp.stdio.env.plugin-root` | Supplied `PLUGIN_ROOT` matches the independently determined package root. | | `mcp.stdio.env.plugin-data-absolute` | Supplied `PLUGIN_DATA` is an absolute path. This check does not exercise writability, storage dedication, or persistence across updates. | | `filesystem.data.writable` | The default server can create, write, and close a small temporary file directly in the supplied `PLUGIN_DATA` directory. | -| `filesystem.data.consistent-within-plugin` | All four servers resolve `PLUGIN_DATA` to the same path. | +| `filesystem.data.consistent-within-plugin` | Servers using omitted and data-rooted working directories resolve `PLUGIN_DATA` to the same filesystem path. | | `mcp.stdio.env.configured-value` | The configured value reaches the subprocess. The fixture does not arrange a conflicting inherited value, so it does not establish override behavior. | | `mcp.stdio.args.preservation-and-expansion` | Argument boundaries, including spaces and an empty argument, are preserved; recognized placeholders expand and unknown placeholder-like text stays literal. | +| `mcp.stdio.command.single-token` | A command path containing a space is preserved as a single executable token. | | `mcp.stdio.env.expansion` | Repeated recognized placeholders expand in environment values and unknown placeholder-like text stays literal. | | `mcp.streamable-http.tool-availability` | The client exposes the HTTP tool and it returns a valid observation despite a conflicting, mixed-case configured `Accept` header. A completed discovery or call attempt with no observation fails this check only when the reporter confirms that the local fixture is healthy. | | `mcp.streamable-http.url.literal-route-and-query` | The configured URL path and query are preserved literally. | diff --git a/plugins/agent-plugins-conformance-core/bin/posix/probe b/plugins/agent-plugins-conformance-core/bin/posix/probe new file mode 100755 index 0000000..843699c --- /dev/null +++ b/plugins/agent-plugins-conformance-core/bin/posix/probe @@ -0,0 +1,2 @@ +#!/bin/sh +exec node "$(dirname "$0")/../../dist/probe.mjs" command-token-posix posix-decoy split "$@" diff --git a/plugins/agent-plugins-conformance-core/bin/posix/probe token.sh b/plugins/agent-plugins-conformance-core/bin/posix/probe token.sh new file mode 100755 index 0000000..47813c3 --- /dev/null +++ b/plugins/agent-plugins-conformance-core/bin/posix/probe token.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exec node "$(dirname "$0")/../../dist/probe.mjs" command-token-posix posix-exact intact "$@" diff --git a/plugins/agent-plugins-conformance-core/bin/windows/probe token.cmd b/plugins/agent-plugins-conformance-core/bin/windows/probe token.cmd new file mode 100644 index 0000000..4a472e1 --- /dev/null +++ b/plugins/agent-plugins-conformance-core/bin/windows/probe token.cmd @@ -0,0 +1,2 @@ +@echo off +node "%~dp0..\..\dist\probe.mjs" command-token-windows windows-exact intact %* diff --git a/plugins/agent-plugins-conformance-core/bin/windows/probe.cmd b/plugins/agent-plugins-conformance-core/bin/windows/probe.cmd new file mode 100644 index 0000000..8ce9f90 --- /dev/null +++ b/plugins/agent-plugins-conformance-core/bin/windows/probe.cmd @@ -0,0 +1,2 @@ +@echo off +node "%~dp0..\..\dist\probe.mjs" command-token-windows windows-decoy split %* diff --git a/plugins/agent-plugins-conformance-core/dist/probe.mjs b/plugins/agent-plugins-conformance-core/dist/probe.mjs index 9a07a87..aeb08c3 100644 --- a/plugins/agent-plugins-conformance-core/dist/probe.mjs +++ b/plugins/agent-plugins-conformance-core/dist/probe.mjs @@ -16967,8 +16967,8 @@ function observeDataWrite(data) { // plugins/agent-plugins-conformance-core/src/probe.mjs var serverName = process.argv[2]; -if (!["default", "relative", "root", "data"].includes(serverName)) { - throw new Error("Expected probe mode: default, relative, root, or data"); +if (!["default", "relative", "root", "data", "command-token-posix", "command-token-windows"].includes(serverName)) { + throw new Error("Expected probe mode: default, relative, root, data, command-token-posix, or command-token-windows"); } var root = realpathSync.native(fileURLToPath(new URL("..", import.meta.url))); var environmentNames = ["PLUGIN_ROOT", "PLUGIN_DATA", "APC_VALUE", "APC_EXPANSION", "APC_LITERAL"]; diff --git a/plugins/agent-plugins-conformance-core/mcp.json b/plugins/agent-plugins-conformance-core/mcp.json index 81442a2..9efec50 100644 --- a/plugins/agent-plugins-conformance-core/mcp.json +++ b/plugins/agent-plugins-conformance-core/mcp.json @@ -56,6 +56,24 @@ ], "cwd": "${PLUGIN_DATA}" }, + "command-token-posix": { + "type": "stdio", + "command": "./bin/posix/probe token.sh", + "args": [ + "arg with spaces", + "", + "literal-value" + ] + }, + "command-token-windows": { + "type": "stdio", + "command": "./bin/windows/probe token.cmd", + "args": [ + "arg with spaces", + "", + "literal-value" + ] + }, "sse": { "type": "sse", "url": "http://127.0.0.1:43187/conformance/sse?value=$APC_SSE_VALUE", diff --git a/plugins/agent-plugins-conformance-core/src/probe.mjs b/plugins/agent-plugins-conformance-core/src/probe.mjs index cd42103..eea9d8a 100644 --- a/plugins/agent-plugins-conformance-core/src/probe.mjs +++ b/plugins/agent-plugins-conformance-core/src/probe.mjs @@ -7,8 +7,8 @@ import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprot import { observeDataWrite } from './data-write.mjs'; const serverName = process.argv[2]; -if (!['default', 'relative', 'root', 'data'].includes(serverName)) { - throw new Error('Expected probe mode: default, relative, root, or data'); +if (!['default', 'relative', 'root', 'data', 'command-token-posix', 'command-token-windows'].includes(serverName)) { + throw new Error('Expected probe mode: default, relative, root, data, command-token-posix, or command-token-windows'); } // Both src/ and dist/ are immediate children of the installed plugin root. // This value is independent of the client-provided PLUGIN_ROOT environment. diff --git a/plugins/agent-plugins-conformance/skills/run-conformance/SKILL.md b/plugins/agent-plugins-conformance/skills/run-conformance/SKILL.md index 0890c33..07e74f3 100644 --- a/plugins/agent-plugins-conformance/skills/run-conformance/SKILL.md +++ b/plugins/agent-plugins-conformance/skills/run-conformance/SKILL.md @@ -69,7 +69,7 @@ Collect all fixture skills and MCP tools through the client's normal mechanisms. } ``` -4. Find the `observe` tools on the Core fixture servers (`default`, `relative`, `root`, `data`, `http`, `http-redirect`, `sse`, `sse-header-precedence`, `sse-redirect`, and `sse-endpoint-origin`) and the Recovery fixture servers (`recovery-valid`, `recovery-cwd-invalid-form`, `recovery-cwd-escape`, `recovery-cwd-data-escape`, `recovery-cwd-symlink-escape`, `recovery-unknown-field`, `recovery-missing-type`, `recovery-env-plugin-root`, `recovery-env-plugin-data`, `recovery-http-type`, `recovery-http-relative-url`, `recovery-http-fragment`, `recovery-http-userinfo`, `recovery-http-duplicate-headers`, `recovery-http-header-name`, `recovery-http-header-value`, `recovery-sse-relative-url`, `recovery-sse-fragment`, `recovery-sse-userinfo`, `recovery-sse-duplicate-headers`, `recovery-sse-header-name`, and `recovery-sse-header-value`) through the client's normal MCP mechanism. Tool names may be client-namespaced. The remote MCP tool descriptions identify their fixture server IDs; retain that association when calling each tool so that an error can be recorded for the same server. Call each available tool with `{}`. First inspect whether the client returned a successful MCP result or a native discovery/call error. For a native remote MCP error, follow step 5 without parsing it as observation JSON. For a native stdio error, leave that server’s runtime observation missing; retain any discovery observation already recorded. For a successful result, take the observation from `structuredContent` (some clients show `structured_content`), or parse the JSON in the tool's text content. Immediately record that complete object; exclude the MCP result wrapper containing `content` or `structuredContent`. Preserve the returned paths and values exactly. The `http-redirect` and `sse-redirect` fixtures redirect requests to another local origin, and `sse-endpoint-origin` advertises a message endpoint on that origin. Do not authorize forwarding configured headers to these destinations. +4. Find the `observe` tools on the Core fixture servers (`default`, `relative`, `root`, `data`, `command-token-posix`, `command-token-windows`, `http`, `http-redirect`, `sse`, `sse-header-precedence`, `sse-redirect`, and `sse-endpoint-origin`) and the Recovery fixture servers (`recovery-valid`, `recovery-cwd-invalid-form`, `recovery-cwd-escape`, `recovery-cwd-data-escape`, `recovery-cwd-symlink-escape`, `recovery-unknown-field`, `recovery-missing-type`, `recovery-env-plugin-root`, `recovery-env-plugin-data`, `recovery-http-type`, `recovery-http-relative-url`, `recovery-http-fragment`, `recovery-http-userinfo`, `recovery-http-duplicate-headers`, `recovery-http-header-name`, `recovery-http-header-value`, `recovery-sse-relative-url`, `recovery-sse-fragment`, `recovery-sse-userinfo`, `recovery-sse-duplicate-headers`, `recovery-sse-header-name`, and `recovery-sse-header-value`) through the client's normal MCP mechanism. Tool names may be client-namespaced. The remote MCP tool descriptions identify their fixture server IDs; retain that association when calling each tool so that an error can be recorded for the same server. Call each available tool with `{}`. First inspect whether the client returned a successful MCP result or a native discovery/call error. For a native remote MCP error, follow step 5 without parsing it as observation JSON. For a native stdio error, leave that server’s runtime observation missing; retain any discovery observation already recorded. Only the platform-appropriate command-token server is expected to run; a startup error for the other is not a collection limitation. For a successful result, take the observation from `structuredContent` (some clients show `structured_content`), or parse the JSON in the tool's text content. Immediately record that complete object; exclude the MCP result wrapper containing `content` or `structuredContent`. Preserve the returned paths and values exactly. The `http-redirect` and `sse-redirect` fixtures redirect requests to another local origin, and `sse-endpoint-origin` advertises a message endpoint on that origin. Do not authorize forwarding configured headers to these destinations. 5. If a completed remote MCP discovery or call attempt instead produces a native error, record it as error evidence: ```json diff --git a/plugins/agent-plugins-conformance/src/cases.mjs b/plugins/agent-plugins-conformance/src/cases.mjs index 00eb183..1cf6bb4 100644 --- a/plugins/agent-plugins-conformance/src/cases.mjs +++ b/plugins/agent-plugins-conformance/src/cases.mjs @@ -26,6 +26,7 @@ export const CASES = Object.freeze([ ['filesystem.data.distinct-across-plugins', 'Distinct data directories across plugins', ['9.1']], ['filesystem.data.consistent-within-plugin', 'Consistent data directory within a plugin', ['9.1']], ['mcp.stdio.env.configured-value', 'Configured environment', ['9.1']], + ['mcp.stdio.command.single-token', 'Command token preservation', ['7.2.1']], ['mcp.stdio.args.preservation-and-expansion', 'Argument preservation and expansion', ['7.2.1', '9.2']], ['mcp.stdio.env.expansion', 'Environment expansion', ['9.2']], ['mcp.streamable-http.tool-availability', 'Streamable HTTP MCP tool evidence', ['6.1', '7.2.1']], diff --git a/plugins/agent-plugins-conformance/src/report.mjs b/plugins/agent-plugins-conformance/src/report.mjs index 5881c16..ca75daa 100644 --- a/plugins/agent-plugins-conformance/src/report.mjs +++ b/plugins/agent-plugins-conformance/src/report.mjs @@ -3,6 +3,10 @@ import { CASES, MCP_CWD_VARIANTS } from './cases.mjs'; export { CASES, CASE_IDS } from './cases.mjs'; const CORE_SERVERS = Object.keys(MCP_CWD_VARIANTS); +const COMMAND_TOKEN_SERVERS = Object.freeze({ + 'command-token-posix': Object.freeze({ exactOrigin: 'posix-exact', decoyOrigin: 'posix-decoy', splitTail: 'token.sh' }), + 'command-token-windows': Object.freeze({ exactOrigin: 'windows-exact', decoyOrigin: 'windows-decoy', splitTail: 'token.cmd' }), +}); const INVALID_STDIO_SERVERS = Object.freeze({ 'recovery-cwd-invalid-form': 'mcp.stdio.cwd.invalid-form', 'recovery-cwd-escape': 'mcp.stdio.cwd.plugin-relative-escape', @@ -34,7 +38,7 @@ const INVALID_STDIO_SERVER_NAMES = Object.keys(INVALID_STDIO_SERVERS); const RECOVERY_INVALID_SERVER_NAMES = [ ...INVALID_STDIO_SERVER_NAMES, ...Object.keys(INVALID_HTTP_SERVERS), ...Object.keys(INVALID_SSE_SERVERS), ]; -const SERVERS = [...CORE_SERVERS, 'recovery-valid', ...INVALID_STDIO_SERVER_NAMES]; +const SERVERS = [...CORE_SERVERS, ...Object.keys(COMMAND_TOKEN_SERVERS), 'recovery-valid', ...INVALID_STDIO_SERVER_NAMES]; const HTTP_SERVERS = ['http', 'http-redirect', ...Object.keys(INVALID_HTTP_SERVERS)]; const SSE_SERVERS = [ 'sse', 'sse-header-precedence', 'sse-redirect', 'sse-endpoint-origin', ...Object.keys(INVALID_SSE_SERVERS), @@ -537,6 +541,34 @@ export function buildReport(input) { 'Working directory matches the resolved expected path.', mismatch('Working directory', target, evidence.cwd)); } } + const defaultEvidence = runtime.get('default'); + const commandTokenId = 'mcp.stdio.command.single-token'; + if (defaultEvidence) { + const defaultFlavor = pathFlavor(defaultEvidence.root); + const platform = defaultFlavor === path.win32 ? 'windows' : 'posix'; + const server = `command-token-${platform}`; + const variant = COMMAND_TOKEN_SERVERS[server]; + const evidence = runtime.get(server); + if (!evidence) { + set(commandTokenId, 'not_verified', `No attributable ${platform} command-token observation was supplied.`); + } else if (!samePath(evidence.root, defaultEvidence.root, defaultFlavor)) { + set(commandTokenId, 'not_verified', + `The ${platform} command-token observation did not identify the same installed plugin root as the Core default observation.`); + } else { + const split = evidence.argv[0] === server && evidence.argv[1] === variant.decoyOrigin && + evidence.argv[2] === 'split' && evidence.argv[3] === variant.splitTail; + const intact = evidence.argv[0] === server && evidence.argv[1] === variant.exactOrigin && evidence.argv[2] === 'intact'; + if (split) { + set(commandTokenId, 'fail', + `The ${platform} decoy wrapper received ${JSON.stringify(variant.splitTail)} as an argument, showing that the configured command was split.`); + } else if (intact) { + set(commandTokenId, 'pass', `The ${platform} exact-name wrapper was selected.`); + } else { + set(commandTokenId, 'not_verified', + `The ${platform} command-token observation did not identify either the exact-name wrapper or an attributable split-name decoy.`); + } + } + } if (skills.has('conformance-recovery-valid')) { check('skills.recovery.valid-skill-available', skills.get('conformance-recovery-valid') === SKILLS['conformance-recovery-valid'], 'Agent reported the expected client-loaded marker for the valid recovery skill.', @@ -630,15 +662,18 @@ export function buildReport(input) { ? `Resolved PLUGIN_DATA unavailable for: ${missingData.join(', ')}.` : `Core servers ${coreData.map(([server]) => server).join(', ')} resolved PLUGIN_DATA to paths distinct from recovery-valid (${JSON.stringify(recoveryData)}).`); - const [firstCoreData, ...otherCoreData] = coreData; - const inconsistentData = firstCoreData - ? otherCoreData.filter(([, data]) => !samePath(data, firstCoreData[1], pathFlavor(firstCoreData[1]))) : []; - set('filesystem.data.consistent-within-plugin', inconsistentData.length ? 'fail' : missingCoreData.length ? 'not_verified' : 'pass', - inconsistentData.length - ? [firstCoreData, ...inconsistentData].map(([server, data]) => `${server} resolved PLUGIN_DATA to ${JSON.stringify(data)}.`).join(' ') - : missingCoreData.length - ? `Resolved PLUGIN_DATA unavailable for: ${missingCoreData.join(', ')}.` - : 'All four core servers resolved PLUGIN_DATA to the same path.'); + const consistencyServers = ['default', 'data']; + const missingConsistencyData = consistencyServers.filter((server) => runtime.get(server)?.resolvedData == null); + if (missingConsistencyData.length) { + set('filesystem.data.consistent-within-plugin', 'not_verified', + `Resolved PLUGIN_DATA unavailable for: ${missingConsistencyData.join(', ')}.`); + } else { + const defaultData = runtime.get('default').resolvedData; + const dataCwdData = runtime.get('data').resolvedData; + check('filesystem.data.consistent-within-plugin', samePath(dataCwdData, defaultData, pathFlavor(defaultData)), + 'The default and data working-directory servers resolved PLUGIN_DATA to the same path.', + `default resolved PLUGIN_DATA to ${JSON.stringify(defaultData)}. data resolved PLUGIN_DATA to ${JSON.stringify(dataCwdData)}.`); + } const evidence = runtime.get('default'); if (evidence) { const { root, env, argv } = evidence; diff --git a/scripts/smoke-codex.mjs b/scripts/smoke-codex.mjs index e147064..1497464 100644 --- a/scripts/smoke-codex.mjs +++ b/scripts/smoke-codex.mjs @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import { spawn, spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; import { closeSync, openSync, writeSync } from 'node:fs'; -import { cp, mkdir, mkdtemp, readFile, readdir, readlink, realpath, rm, writeFile } from 'node:fs/promises'; +import { cp, mkdir, mkdtemp, readFile, readdir, readlink, realpath, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { delimiter, dirname, isAbsolute, join, relative } from 'node:path'; import { createInterface } from 'node:readline'; @@ -16,7 +16,9 @@ const [codex, ...extra] = process.argv.slice(2); assert.ok(codex && isAbsolute(codex) && extra.length === 0, 'Usage: node scripts/smoke-codex.mjs '); const names = ['agent-plugins-conformance-core', 'agent-plugins-conformance', 'agent-plugins-conformance-recovery']; -const servers = ['default', 'relative', 'root', 'data', 'http', 'recovery-valid']; +const commandPlatform = process.platform === 'win32' ? 'windows' : 'posix'; +const commandServer = `command-token-${commandPlatform}`; +const servers = ['default', 'relative', 'root', 'data', commandServer, 'http', 'recovery-valid']; const invalidSseServers = [ 'recovery-sse-relative-url', 'recovery-sse-fragment', 'recovery-sse-userinfo', 'recovery-sse-duplicate-headers', 'recovery-sse-header-name', 'recovery-sse-header-value', @@ -24,7 +26,7 @@ const invalidSseServers = [ // Redirect refusal requires the guiding agent to interpret and preserve the native error. const coveredCases = CASE_IDS.filter((id) => (id.startsWith('mcp.') || id.startsWith('filesystem.')) && id !== 'mcp.streamable-http.headers.cross-origin-redirect' && !id.startsWith('mcp.sse.')); -const temporary = await realpath(await mkdtemp(join(tmpdir(), 'apc-codex-smoke-'))); +const temporary = await realpath(await mkdtemp(join(tmpdir(), 'apc codex smoke '))); const home = join(temporary, 'codex-home'); const workspace = join(temporary, 'workspace'); const marketplace = join(temporary, 'marketplace'); @@ -161,6 +163,12 @@ try { if (!actual.some(([path]) => path === 'escape-link')) expected = original.filter(([path]) => path !== 'escape-link'); } assert.deepEqual(actual, expected, `${name}: installed package differs from source`); + if (name === 'agent-plugins-conformance-core' && process.platform !== 'win32') { + for (const file of ['bin/posix/probe token.sh', 'bin/posix/probe']) { + assert.ok((await stat(join(installedPath, file))).mode & 0o111, + `${file}: installed wrapper is not executable`); + } + } installed.push(installedPath); } const reporter = join(installed[1], 'skills/run-conformance/scripts/report.mjs'); @@ -226,6 +234,11 @@ try { : invalidSseServers.includes(server) ? 'mcp-sse' : 'mcp-stdio', `${server}: missing observation`); assert.equal(observation.server, server, `${server}: unexpected observation server`); + if (server === commandServer) { + assert.deepEqual(observation.evidence.argv, + [commandServer, `${commandPlatform}-exact`, 'intact', 'arg with spaces', '', 'literal-value'], + 'Native launch did not preserve the command token and configured arguments'); + } record({ action: 'record', observation }); } // SSE is optional. Preserve native diagnostics even when the loader omits entries. diff --git a/test/command-token.test.mjs b/test/command-token.test.mjs new file mode 100644 index 0000000..c447fda --- /dev/null +++ b/test/command-token.test.mjs @@ -0,0 +1,114 @@ +import assert from 'node:assert/strict'; +import { cp, lstat, mkdtemp, readFile, realpath, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import test from 'node:test'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; + +const configuredArgs = ['arg with spaces', '', 'literal-value']; + +async function fixture(t) { + const parent = await mkdtemp(join(tmpdir(), 'apc command token ')); + const root = join(await realpath(parent), 'installed plugin with spaces'); + const clients = []; + const protocolErrors = []; + await cp(new URL('../plugins/agent-plugins-conformance-core/', import.meta.url), root, { recursive: true }); + t.after(async () => { + try { + const results = await Promise.allSettled(clients.map((client) => client.close())); + const failures = results.filter(({ status }) => status === 'rejected').map(({ reason }) => reason); + if (failures.length) throw new AggregateError(failures, 'MCP clients failed to close'); + assert.deepEqual(protocolErrors, [], 'The server wrote invalid MCP output'); + } finally { + await rm(parent, { recursive: true, force: true }); + } + }); + const config = JSON.parse(await readFile(join(root, 'mcp.json'), 'utf8')); + return { parent, root, clients, config, protocolErrors }; +} + +async function connect(files, server, { split = false, batch = false } = {}) { + const configured = files.config.mcpServers[server]; + let command = resolve(files.root, configured.command.slice(2)); + let args = configured.args; + if (split) { + if (process.platform === 'win32') { + command = join(files.parent, 'bad command token launcher.cmd'); + await writeFile(command, '@echo off\r\n.\\bin\\windows\\probe token.cmd %*\r\n'); + } else { + command = '/bin/sh'; + args = ['-c', `${configured.command} "$@"`, 'apc-command-token', ...configured.args]; + } + } + if (batch) { + // Preserve ordinary batch echo settings; the fixture must keep stdout protocol-clean. + command = join(files.parent, 'ordinary batch launcher.cmd'); + await writeFile(command, '@call ".\\bin\\windows\\probe token.cmd" %*\r\n'); + } + const env = Object.fromEntries(Object.entries(process.env)); + for (const name of ['PLUGIN_ROOT', 'PLUGIN_DATA', 'APC_VALUE', 'APC_EXPANSION', 'APC_LITERAL']) delete env[name]; + const client = new Client({ name: 'command-token-reference-test', version: '1' }); + client.onerror = (error) => files.protocolErrors.push(error); + files.clients.push(client); + await client.connect(new StdioClientTransport({ command, args, cwd: files.root, env, stderr: 'pipe' })); + return client; +} + +async function observe(client) { + const { tools } = await client.listTools(); + assert.deepEqual(tools.map(({ name }) => name), ['observe']); + const result = await client.callTool({ name: 'observe', arguments: {} }); + assert.equal(result.isError, undefined); + assert.deepEqual(JSON.parse(result.content[0].text), result.structuredContent); + return result.structuredContent; +} + +test('the copied Core fixture preserves the platform command token and configured arguments', + { timeout: 30_000 }, async (t) => { + const files = await fixture(t); + assert.equal((await readdir(files.root)).includes('node_modules'), false); + const platform = process.platform === 'win32' ? 'windows' : 'posix'; + const server = `command-token-${platform}`; + const observation = await observe(await connect(files, server)); + assert.equal(observation.kind, 'mcp-stdio'); + assert.equal(observation.server, server); + assert.equal(observation.evidence.server, server); + assert.equal(observation.evidence.root, await realpath(files.root)); + assert.equal(observation.evidence.cwd, await realpath(files.root)); + assert.equal(observation.evidence.resolvedData, null); + assert.deepEqual(observation.evidence.env, {}); + assert.deepEqual(observation.evidence.argv, + [server, `${platform}-exact`, 'intact', ...configuredArgs]); + }); + +test('deliberate unquoted platform-shell parsing reaches the split-name decoy', + { timeout: 30_000 }, async (t) => { + const files = await fixture(t); + const platform = process.platform === 'win32' ? 'windows' : 'posix'; + const tail = process.platform === 'win32' ? 'token.cmd' : 'token.sh'; + const server = `command-token-${platform}`; + const observation = await observe(await connect(files, server, { split: true })); + assert.equal(observation.kind, 'mcp-stdio'); + assert.equal(observation.server, server); + assert.deepEqual(observation.evidence.argv, + [server, `${platform}-decoy`, 'split', tail, ...configuredArgs]); + }); + +test('explicit Windows batch execution preserves the command and arguments without non-MCP stdout', + { skip: process.platform !== 'win32', timeout: 30_000 }, async (t) => { + const files = await fixture(t); + const server = 'command-token-windows'; + const observation = await observe(await connect(files, server, { batch: true })); + assert.deepEqual(observation.evidence.argv, + [server, 'windows-exact', 'intact', ...configuredArgs]); + }); + +test('Core packages paired native wrappers with the intended executable modes', + { skip: process.platform === 'win32' }, async () => { + const root = new URL('../plugins/agent-plugins-conformance-core/bin/', import.meta.url); + assert.notEqual((await lstat(new URL('posix/probe token.sh', root))).mode & 0o111, 0); + assert.notEqual((await lstat(new URL('posix/probe', root))).mode & 0o111, 0); + assert.equal((await lstat(new URL('windows/probe token.cmd', root))).mode & 0o111, 0); + assert.equal((await lstat(new URL('windows/probe.cmd', root))).mode & 0o111, 0); + }); diff --git a/test/probe.test.mjs b/test/probe.test.mjs index 19f70db..d2af31c 100644 --- a/test/probe.test.mjs +++ b/test/probe.test.mjs @@ -94,7 +94,7 @@ test('copied plugin runs only MCP observation tools without node_modules', { tim const direct = buildReport(reportInput); assert.equal(direct.summary.fail, 0); assert.equal(direct.summary.pass, 15); - assert.equal(direct.summary.not_verified, 37); + assert.equal(direct.summary.not_verified, 38); }); test('copied recovery plugin serves exact valid and invalid-server observations without runtime dependencies', { timeout: 30_000 }, async (t) => { diff --git a/test/record-report.test.mjs b/test/record-report.test.mjs index dc38c82..4c7f44f 100644 --- a/test/record-report.test.mjs +++ b/test/record-report.test.mjs @@ -23,6 +23,21 @@ const mcp = () => ({ }, }, }); +const commandToken = (platform = 'posix', { + root = platform === 'posix' ? '/plugin' : 'C:\\plugin', + data = platform === 'posix' ? '/data' : 'D:\\data', + origin = `${platform}-exact`, marker = 'intact', + args = ['arg with spaces', '', 'literal-value'], +} = {}) => { + const server = `command-token-${platform}`; + return { + kind: 'mcp-stdio', server, + evidence: { + version: 1, server, root, cwd: root, resolvedData: data, + argv: [server, origin, marker, ...args], env: {}, + }, + }; +}; const recoverySkill = (marker = 'APC_RECOVERY_VALID_V1') => ({ kind: 'skill', skill: 'conformance-recovery-valid', marker, }); @@ -376,6 +391,34 @@ test('distinct keys accumulate in deterministic order; repeated keys fully repla assert.deepEqual(await readdir(join(f.directory, 'nested directory')), ['report with spaces.json']); }); +test('command-token evidence records canonically and replacement changes only its verdict', async (t) => { + const f = await fixture(t); + f.success(start); + f.record(mcp()); + const exact = commandToken(); + f.record(exact); + let report = await f.read(); + assert.equal(report.results.find(({ id }) => id === 'mcp.stdio.command.single-token').status, 'pass'); + assert.deepEqual(report.observations, [mcp(), exact]); + + exact.evidence.argv.push('changed after recording'); + assert.notDeepEqual(report.observations[1], exact); + + const split = commandToken('posix', { + origin: 'posix-decoy', marker: 'split', args: ['token.sh', 'mangled', 'later', 'arguments'], + }); + f.record(split); + report = await f.read(); + assert.equal(report.results.find(({ id }) => id === 'mcp.stdio.command.single-token').status, 'fail'); + assert.deepEqual(report.observations, [mcp(), split]); + + const windows = commandToken('windows'); + f.record(windows); + report = await f.read(); + assert.equal(report.results.find(({ id }) => id === 'mcp.stdio.command.single-token').status, 'fail'); + assert.deepEqual(report.observations, [mcp(), split, windows]); +}); + test('recovery observations have distinct recorder keys and repeated keys replace canonically', async (t) => { const f = await fixture(t); f.success(start); diff --git a/test/report-human.test.mjs b/test/report-human.test.mjs index 53acefb..be5898f 100644 --- a/test/report-human.test.mjs +++ b/test/report-human.test.mjs @@ -21,6 +21,13 @@ function input() { APC_EXPANSION: `${root}|${data}|${root}`, APC_LITERAL: '${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}' }, }, })), + { + kind: 'mcp-stdio', server: 'command-token-posix', + evidence: { + version: 1, server: 'command-token-posix', root, cwd: root, resolvedData: data, + argv: ['command-token-posix', 'posix-exact', 'intact', 'arg with spaces', '', 'literal-value'], env: {}, + }, + }, { kind: 'skill-discovery', skill: 'conformance-nested', advertised: false }, ], }; @@ -52,9 +59,9 @@ test('complete stdio and recovery evidence leaves optional HTTP and SSE checks u '', 'Checks Passed Failed Not verified', 'Skills 3 0 0', - 'MCP 21 0 24', + 'MCP 22 0 24', 'Filesystem 4 0 0', - 'Total 28 0 24', + 'Total 29 0 24', ].join('\n')); const missing = human.split('\n\nNot verified\n')[1]; for (const id of [ @@ -91,7 +98,7 @@ test('all-unverified human report preserves every saved missing-evidence result' value.observations = []; const human = formatReport(buildReport(value)); assert.match(human, /Skills\s+0\s+0\s+3/); - assert.match(human, /MCP\s+0\s+0\s+45/); + assert.match(human, /MCP\s+0\s+0\s+46/); assert.match(human, /Filesystem\s+0\s+0\s+4/); assert.match(human, /Missing skill observations: conformance-alpha, conformance-beta\./); for (const result of buildReport(value).results) assert.ok(human.includes(`(${result.id})`)); @@ -103,7 +110,7 @@ test('mixed human report puts failures before missing-evidence details', () => { value.observations = [value.observations[2]]; value.observations[0].evidence.env.APC_VALUE = 'incorrect configured value'; const human = formatReport(buildReport(value)); - assert.match(human, /MCP\s+6\s+1\s+38/); + assert.match(human, /MCP\s+6\s+1\s+39/); assert.match(human, /Filesystem\s+1\s+0\s+3/); assert.match(human, /Configured environment \(mcp.stdio.env.configured-value\)/); assert.match(human, /"fixture value with spaces"/); @@ -121,8 +128,10 @@ test('unverified checks with supplied observations keep their individual reasons assert.match(human, /Argument preservation and expansion \(mcp.stdio.args.preservation-and-expansion\)/); assert.match(human, /Environment expansion \(mcp.stdio.env.expansion\)/); assert.match(human, /data working directory \(mcp.stdio.cwd.plugin-data\)/); + assert.match(human, /Consistent data directory within a plugin \(filesystem.data.consistent-within-plugin\)/); assert.match(human, /PLUGIN_DATA is missing; expected expansion cannot be computed\./); assert.match(human, /PLUGIN_DATA could not be resolved/); + assert.match(human, /Resolved PLUGIN_DATA unavailable for: data\./); }); test('human diagnostic values cannot inject terminal controls or fake lines', () => { @@ -158,7 +167,7 @@ test('human report renders saved warnings independently of result status', () => const writable = value.results.find(({ id }) => id === 'filesystem.data.writable'); writable.status = 'not_verified'; writable.warning = 'Saved cleanup warning for /state/plugin/leftover.'; - value.summary = { pass: 15, fail: 0, not_verified: 37, total: 52 }; + value.summary = { pass: 16, fail: 0, not_verified: 37, total: 53 }; value.observations = []; const human = formatReport(value); diff --git a/test/report.test.mjs b/test/report.test.mjs index 797585c..4bb7d2a 100644 --- a/test/report.test.mjs +++ b/test/report.test.mjs @@ -5,6 +5,7 @@ import { buildReport, CASE_IDS } from '../plugins/agent-plugins-conformance/src/ function input(root = '/fixture/plugin', data = '/state/plugin') { const flavor = root.startsWith('/') ? path.posix : path.win32; + const platform = flavor === path.win32 ? 'windows' : 'posix'; return { schemaVersion: 1, observations: [ @@ -21,9 +22,22 @@ function input(root = '/fixture/plugin', data = '/state/plugin') { APC_EXPANSION: `${root}|${data}|${root}`, APC_LITERAL: '${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}' }, }, })), + commandTokenObservation(platform, root, data), ], }; } +const commandTokenObservation = (platform, root = '/fixture/plugin', data = '/state/plugin', { + origin = `${platform}-exact`, marker = 'intact', args = ['arg with spaces', '', 'literal-value'], +} = {}) => { + const server = `command-token-${platform}`; + return { + kind: 'mcp-stdio', server, + evidence: { + version: 1, server, root, cwd: root, resolvedData: data, + argv: [server, origin, marker, ...args], env: {}, + }, + }; +}; const runtime = (value, server = 'default') => value.observations.find((observation) => observation.server === server).evidence; const result = (report, id) => report.results.find((item) => item.id === id); const recoveryObservations = () => [ @@ -71,7 +85,7 @@ const invalidSseServerCases = [ test('complete stdio and skill core evidence passes its cases and preserves canonical evidence', () => { const value = input(); const report = buildReport(value); - assert.deepEqual(report.summary, { pass: 16, fail: 0, not_verified: 36, total: 52 }); + assert.deepEqual(report.summary, { pass: 17, fail: 0, not_verified: 36, total: 53 }); assert.deepEqual(report.results.map(({ id }) => id), CASE_IDS); assert.equal(report.specVersion, '1.0.0'); assert.deepEqual(result(report, 'mcp.stdio.env.plugin-root').specSections, ['9.1']); @@ -84,19 +98,19 @@ test('complete stdio and skill core evidence passes its cases and preserves cano test('recovery witnesses extend the canonical report without changing core-only results', () => { const coreOnly = buildReport(input()); - assert.deepEqual(coreOnly.summary, { pass: 16, fail: 0, not_verified: 36, total: 52 }); + assert.deepEqual(coreOnly.summary, { pass: 17, fail: 0, not_verified: 36, total: 53 }); assert.equal(result(coreOnly, 'skills.recovery.valid-skill-available').status, 'not_verified'); assert.equal(result(coreOnly, 'mcp.stdio.recovery.valid-server-available').status, 'not_verified'); const recoveryOnly = buildReport({ schemaVersion: 1, observations: recoveryObservations() }); - assert.deepEqual(recoveryOnly.summary, { pass: 2, fail: 0, not_verified: 50, total: 52 }); + assert.deepEqual(recoveryOnly.summary, { pass: 2, fail: 0, not_verified: 51, total: 53 }); assert.equal(result(recoveryOnly, 'skills.recovery.valid-skill-available').status, 'pass'); assert.equal(result(recoveryOnly, 'mcp.stdio.recovery.valid-server-available').status, 'pass'); const combinedInput = input(); combinedInput.observations.push(...recoveryObservations()); const combined = buildReport(combinedInput); - assert.deepEqual(combined.summary, { pass: 19, fail: 0, not_verified: 33, total: 52 }); + assert.deepEqual(combined.summary, { pass: 20, fail: 0, not_verified: 33, total: 53 }); assert.deepEqual(combined.results.map(({ id }) => id), CASE_IDS); const recoveryIds = new Set([ 'skills.recovery.valid-skill-available', @@ -119,7 +133,7 @@ test('recovery witnesses extend the canonical report without changing core-only assert.equal(JSON.stringify(buildReport(reordered)), JSON.stringify(combined)); const missing = buildReport({ schemaVersion: 1, observations: [] }); - assert.deepEqual(missing.summary, { pass: 0, fail: 0, not_verified: 52, total: 52 }); + assert.deepEqual(missing.summary, { pass: 0, fail: 0, not_verified: 53, total: 53 }); }); test('an incorrect recovery skill marker fails its availability check independently', () => { @@ -130,7 +144,7 @@ test('an incorrect recovery skill marker fails its availability check independen assert.equal(availability.status, 'fail'); assert.match(availability.detail, /expected "APC_RECOVERY_VALID_V1"; observed "wrong recovery marker"/); assert.equal(result(report, 'mcp.stdio.recovery.valid-server-available').status, 'pass'); - assert.deepEqual(report.summary, { pass: 1, fail: 1, not_verified: 50, total: 52 }); + assert.deepEqual(report.summary, { pass: 1, fail: 1, not_verified: 51, total: 53 }); }); test('report bytes are deterministic across observation and property orders', () => { @@ -145,11 +159,11 @@ test('missing observations remain unverified and every result identifies its hie const value = input(); value.observations = []; const report = buildReport(value); - assert.deepEqual(report.summary, { pass: 0, fail: 0, not_verified: 52, total: 52 }); + assert.deepEqual(report.summary, { pass: 0, fail: 0, not_verified: 53, total: 53 }); const skills = report.results.filter(({ id }) => id.startsWith('skills.')); const mcp = report.results.filter(({ id }) => id.startsWith('mcp.')); assert.deepEqual(skills.map(({ id }) => id), ['skills.discovery.immediate-children', 'skills.recovery.valid-skill-available', 'skills.recovery.invalid-mcp-document']); - assert.equal(mcp.length, 45); + assert.equal(mcp.length, 46); assert.equal(report.results.filter(({ id }) => id.startsWith('filesystem.')).length, 4); assert.ok(mcp.some(({ id }) => id === 'mcp.stdio.env.plugin-root')); for (const result of report.results) { @@ -222,7 +236,7 @@ test('data cwd follows resolved aliases while expansion preserves the original e runtime(value).dataWrite.path = '/private/tmp/data/.agent-plugins-conformance-write-test'; runtime(value, 'data').cwd = '/private/tmp/data'; const report = buildReport(value); - assert.equal(report.summary.pass, 16); + assert.equal(report.summary.pass, 17); assert.equal(runtime(report, 'data').resolvedData, '/private/tmp/data'); assert.equal(runtime(report).env.PLUGIN_DATA, '/tmp/data'); runtime(value, 'data').resolvedData = null; @@ -256,7 +270,7 @@ test('data paths must remain distinct across the recovery and complete core evid assert.equal(result(buildReport(windows), id).status, 'fail'); }); -test('core probes require one consistent resolved data path, under POSIX and Windows rules', () => { +test('default and data cwd probes require one consistent resolved data path, under POSIX and Windows rules', () => { const id = 'filesystem.data.consistent-within-plugin'; for (const [root, data, different] of [ ['/fixture/plugin', '/state/plugin', '/state/other'], @@ -265,14 +279,28 @@ test('core probes require one consistent resolved data path, under POSIX and Win const value = input(root, data); assert.equal(result(buildReport(value), id).status, 'pass'); - const mismatch = structuredClone(value); - mismatch.observations = mismatch.observations.filter(({ server }) => !server || server === 'default' || server === 'data'); + const requiredOnly = structuredClone(value); + requiredOnly.observations = requiredOnly.observations.filter(({ server }) => !server || server === 'default' || server === 'data'); + assert.equal(result(buildReport(requiredOnly), id).status, 'pass'); + + const mismatch = structuredClone(requiredOnly); runtime(mismatch, 'data').resolvedData = different; assert.equal(result(buildReport(mismatch), id).status, 'fail'); - const incomplete = structuredClone(value); - incomplete.observations = incomplete.observations.filter(({ server }) => !server || server !== 'data'); - assert.equal(result(buildReport(incomplete), id).status, 'not_verified'); + for (const missingServer of ['default', 'data']) { + const absent = structuredClone(requiredOnly); + absent.observations = absent.observations.filter(({ server }) => server !== missingServer); + assert.equal(result(buildReport(absent), id).status, 'not_verified'); + + const unresolved = structuredClone(requiredOnly); + runtime(unresolved, missingServer).resolvedData = null; + assert.equal(result(buildReport(unresolved), id).status, 'not_verified'); + } + + const unrelated = structuredClone(value); + runtime(unrelated, 'relative').resolvedData = different; + runtime(unrelated, 'root').resolvedData = null; + assert.equal(result(buildReport(unrelated), id).status, 'pass'); } }); @@ -306,10 +334,78 @@ test('arguments preserve empty values, literal placeholders, spaces, and exact o } }); +test('the platform command fixture establishes exact execution and attributable splitting', () => { + const id = 'mcp.stdio.command.single-token'; + for (const [root, data, platform, tail] of [ + ['/fixture/plugin', '/state/plugin', 'posix', 'token.sh'], + ['C:\\fixture\\plugin', 'D:\\state\\plugin', 'windows', 'token.cmd'], + ]) { + const value = input(root, data); + const exact = result(buildReport(value), id); + assert.equal(exact.status, 'pass'); + assert.deepEqual(exact.specSections, ['7.2.1']); + assert.match(exact.detail, new RegExp(`${platform} exact-name wrapper`)); + + const observation = value.observations.find(({ server }) => server === `command-token-${platform}`); + observation.evidence.argv = [observation.server, `${platform}-exact`, 'intact', 'mangled', 'later', 'arguments']; + assert.equal(result(buildReport(value), id).status, 'pass'); + observation.evidence.argv = [observation.server, `${platform}-decoy`, 'split', tail, 'mangled', 'later', 'arguments']; + const split = result(buildReport(value), id); + assert.equal(split.status, 'fail'); + assert.match(split.detail, new RegExp(`${platform} decoy wrapper`)); + assert.match(split.detail, new RegExp(tail.replace('.', '\\.'))); + } +}); + +test('command-token attribution requires independent matching Core root evidence', () => { + const id = 'mcp.stdio.command.single-token'; + const core = input(); + const defaultObservation = core.observations.find(({ server }) => server === 'default'); + const posixObservation = core.observations.find(({ server }) => server === 'command-token-posix'); + const windowsObservation = commandTokenObservation('windows', 'C:\\fixture\\plugin', 'D:\\state\\plugin'); + + for (const observations of [ + [posixObservation], + [defaultObservation], + [defaultObservation, windowsObservation], + ]) { + assert.equal(result(buildReport({ schemaVersion: 1, observations }), id).status, 'not_verified'); + } + + const mismatched = structuredClone(posixObservation); + mismatched.evidence.root = '/different/plugin'; + assert.match(result(buildReport({ schemaVersion: 1, observations: [defaultObservation, mismatched] }), id).detail, + /same installed plugin root/); + + const unattributable = structuredClone(posixObservation); + unattributable.evidence.argv = [unattributable.server, 'posix-decoy', 'split', 'not-the-command-tail']; + assert.match(result(buildReport({ schemaVersion: 1, observations: [defaultObservation, unattributable] }), id).detail, + /did not identify either/); + + assert.throws(() => buildReport({ schemaVersion: 1, observations: [posixObservation, structuredClone(posixObservation)] }), + /duplicate mcp-stdio observation: command-token-posix/); +}); + +test('applicable split evidence fails despite a wrong-platform exact observation', () => { + const value = input(); + const posix = value.observations.find(({ server }) => server === 'command-token-posix'); + posix.evidence.argv = [posix.server, 'posix-decoy', 'split', 'token.sh']; + value.observations.push(commandTokenObservation('windows', 'C:\\fixture\\plugin', 'D:\\state\\plugin')); + assert.equal(result(buildReport(value), 'mcp.stdio.command.single-token').status, 'fail'); +}); + +test('command-token observations stay outside cwd and plugin-data aggregate checks', () => { + const observation = commandTokenObservation('posix'); + const report = buildReport({ schemaVersion: 1, observations: [observation] }); + assert.equal(result(report, 'mcp.stdio.command.single-token').status, 'not_verified'); + assert.ok(report.results.filter(({ id }) => id !== 'mcp.stdio.command.single-token') + .every(({ status }) => status === 'not_verified')); +}); + test('path comparisons follow producing OS and normalize path components', () => { for (const [root, data] of [['/plugin with spaces', '/data with spaces'], ['C:\\plugin', 'D:\\data'], ['\\\\host\\share\\plugin', '\\\\host\\share\\data']]) { const value = input(root, data); - assert.equal(buildReport(value).summary.pass, 16); + assert.equal(buildReport(value).summary.pass, 17); const flavor = root.startsWith('/') ? path.posix : path.win32; runtime(value).cwd = `${root}${flavor.sep}sub${flavor.sep}..`; assert.equal(result(buildReport(value), 'mcp.stdio.cwd.omitted').status, 'pass'); @@ -565,7 +661,7 @@ test('malformed MCP skill evidence affects only its own result for missing, corr assert.deepEqual(result(report, id).specSections, ['7.2.2']); assert.deepEqual(report.results.filter((item) => item.id !== id), baseline.results.filter((item) => item.id !== id)); assert.deepEqual(report.observations.find(({ skill }) => skill === observation.skill), observation); - assert.deepEqual(report.summary, { pass: status === 'pass' ? 20 : 19, fail: status === 'fail' ? 1 : 0, not_verified: 32, total: 52 }); + assert.deepEqual(report.summary, { pass: status === 'pass' ? 21 : 20, fail: status === 'fail' ? 1 : 0, not_verified: 32, total: 53 }); if (status === 'fail') assert.match(result(report, id).detail, /expected "APC_INVALID_MCP_VALID_V1"; observed "incorrect marker"/); const reversed = { schemaVersion: 1, observations: [...value.observations, observation].reverse() }; assert.equal(JSON.stringify(buildReport(reversed)), JSON.stringify(report));