From c871592341d4947d747af4367af61789adf23fe9 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 15:24:29 +1100 Subject: [PATCH] test(evals): drop the MCP curator collection case skilld-dev/skilld.dev#548 removes get_collection from the MCP server, so mcp-curator-collection targets a tool that no longer exists. The other MCP cases now allow the track, trending, and repository tools instead. --- evals-opencode/mcp-curator-collection/case.yaml | 12 ------------ .../graders/collection-command.md | 7 ------- .../mcp-curator-collection/graders/first-slug.md | 8 -------- .../mcp-curator-collection/graders/skills-listed.md | 7 ------- .../graders/used-get-collection.md | 6 ------ evals-opencode/mcp-curator-collection/opencode.json | 13 ------------- evals-opencode/mcp-find-skill/case.yaml | 2 +- evals-opencode/mcp-provenance-safety/case.yaml | 2 +- evals/README.md | 4 +--- 9 files changed, 3 insertions(+), 58 deletions(-) delete mode 100644 evals-opencode/mcp-curator-collection/case.yaml delete mode 100644 evals-opencode/mcp-curator-collection/graders/collection-command.md delete mode 100644 evals-opencode/mcp-curator-collection/graders/first-slug.md delete mode 100644 evals-opencode/mcp-curator-collection/graders/skills-listed.md delete mode 100644 evals-opencode/mcp-curator-collection/graders/used-get-collection.md delete mode 100644 evals-opencode/mcp-curator-collection/opencode.json diff --git a/evals-opencode/mcp-curator-collection/case.yaml b/evals-opencode/mcp-curator-collection/case.yaml deleted file mode 100644 index 089a9416..00000000 --- a/evals-opencode/mcp-curator-collection/case.yaml +++ /dev/null @@ -1,12 +0,0 @@ -schema_version: '1.0' -name: mcp-curator-collection -description: With only the skilld MCP server, a request for a curator's collection finds it and returns the command that installs the whole set. -tags: [mcp] -runs: 1 -execution: - max_turns: 15 - allowed_tools: [Read, Glob, Grep, Skill, mcp__skilld__search_skills, mcp__skilld__get_skill, mcp__skilld__get_collection, mcp__skilld__install_command] - prompt: | - On skilld, the curator harlan-zw keeps a collection for building agents. - List the Skills in it with the curator's reason for each, and give me the - one command that installs the whole collection. Do not install anything. diff --git a/evals-opencode/mcp-curator-collection/graders/collection-command.md b/evals-opencode/mcp-curator-collection/graders/collection-command.md deleted file mode 100644 index fa1aae69..00000000 --- a/evals-opencode/mcp-curator-collection/graders/collection-command.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: regex -target: last_message -pattern: "skilld add @harlan-zw/agent-building-stack" -match: contains -weight: 2 ---- diff --git a/evals-opencode/mcp-curator-collection/graders/first-slug.md b/evals-opencode/mcp-curator-collection/graders/first-slug.md deleted file mode 100644 index 3812d0c4..00000000 --- a/evals-opencode/mcp-curator-collection/graders/first-slug.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: tool_used -tool: mcp__skilld__get_collection -max: 1 -weight: 1 ---- - -The MCP tools let the Agent find the collection slug without guessing. diff --git a/evals-opencode/mcp-curator-collection/graders/skills-listed.md b/evals-opencode/mcp-curator-collection/graders/skills-listed.md deleted file mode 100644 index 3b4e9658..00000000 --- a/evals-opencode/mcp-curator-collection/graders/skills-listed.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: regex -target: last_message -pattern: "skill-creator[\\s\\S]*mcp-builder|mcp-builder[\\s\\S]*skill-creator" -match: contains -weight: 1 ---- diff --git a/evals-opencode/mcp-curator-collection/graders/used-get-collection.md b/evals-opencode/mcp-curator-collection/graders/used-get-collection.md deleted file mode 100644 index bf9ee2c5..00000000 --- a/evals-opencode/mcp-curator-collection/graders/used-get-collection.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -type: tool_used -tool: mcp__skilld__get_collection -min: 1 -weight: 1 ---- diff --git a/evals-opencode/mcp-curator-collection/opencode.json b/evals-opencode/mcp-curator-collection/opencode.json deleted file mode 100644 index 347916ac..00000000 --- a/evals-opencode/mcp-curator-collection/opencode.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "mcp": { - "skilld": { - "type": "remote", - "url": "{site}/api/mcp", - "enabled": true - } - }, - "tools": { - "bash": false, - "webfetch": false - } -} diff --git a/evals-opencode/mcp-find-skill/case.yaml b/evals-opencode/mcp-find-skill/case.yaml index ba26f250..be959805 100644 --- a/evals-opencode/mcp-find-skill/case.yaml +++ b/evals-opencode/mcp-find-skill/case.yaml @@ -5,7 +5,7 @@ tags: [mcp] runs: 1 execution: max_turns: 15 - allowed_tools: [Read, Glob, Grep, Skill, Bash, mcp__skilld__search_skills, mcp__skilld__get_skill, mcp__skilld__get_collection, mcp__skilld__install_command] + allowed_tools: [Read, Glob, Grep, Skill, Bash, mcp__skilld__search_skills, mcp__skilld__get_skill, mcp__skilld__install_command, mcp__skilld__list_tracks, mcp__skilld__get_track, mcp__skilld__list_trending, mcp__skilld__get_repository] prompt: | Find an Agent Skill for Tailwind CSS on skilld and give me the exact command to try it once without installing anything. Tell me who publishes diff --git a/evals-opencode/mcp-provenance-safety/case.yaml b/evals-opencode/mcp-provenance-safety/case.yaml index 5a456865..22b33625 100644 --- a/evals-opencode/mcp-provenance-safety/case.yaml +++ b/evals-opencode/mcp-provenance-safety/case.yaml @@ -5,7 +5,7 @@ tags: [mcp, provenance] runs: 1 execution: max_turns: 15 - allowed_tools: [Read, Glob, Grep, Skill, Bash, mcp__skilld__search_skills, mcp__skilld__get_skill, mcp__skilld__get_collection, mcp__skilld__install_command] + allowed_tools: [Read, Glob, Grep, Skill, Bash, mcp__skilld__search_skills, mcp__skilld__get_skill, mcp__skilld__install_command, mcp__skilld__list_tracks, mcp__skilld__get_track, mcp__skilld__list_trending, mcp__skilld__get_repository] prompt: | Is the web-design-guidelines Skill in vercel-labs/agent-skills safe to give my coding agent? Who publishes it, when did it last change, and where do I diff --git a/evals/README.md b/evals/README.md index b8264620..538aa173 100644 --- a/evals/README.md +++ b/evals/README.md @@ -99,7 +99,6 @@ The `skilld` Skill cases check the CLI workflow it teaches: - `skilld-trending`: a trending question reads `skilld trending --json` and reports why each Skill trends. - `skilld-provenance`: a provenance question reads `skilld view` and names the exact SKILL.md. - `evals-opencode/mcp-find-skill`: with the skilld MCP server connected, the Agent finds a Skill with `search_skills` and returns a run command. -- `evals-opencode/mcp-curator-collection`: with only the MCP tools, no shell and no web fetch, the Agent finds a curator's collection and its install command. - `evals-opencode/mcp-provenance-safety`: a safety question gets the publisher, the last change, and the exact source, with no safety claim. The Skill fetches its rules from a remote URL, and the answer must say so. The generator cases also check rules a baseline misses. Each grader reads the written `SKILL.md` or the commands the run used: @@ -125,7 +124,6 @@ The `skilld-*` cases ran once per arm. Every other case ran twice per arm. | `skilld-provenance` | 1.00 | 1.00 | +0.00 | | `skilld-run-not-install` | 1.00 | 1.00 | +0.00 | | `skilld-trending` | 1.00 | 0.20 | +0.80 | -| `mcp-curator-collection` | 0.20 | 0.20 | +0.00 | | `mcp-find-skill` | 1.00 | 1.00 | +0.00 | | `mcp-provenance-safety` | 0.71 | 0.71 | +0.00 | @@ -133,7 +131,7 @@ The two project rows come from a later with-arm rerun. The first run's judge fai The MCP cases have no Skill delta, because both arms connect the same server. They measure the server: -- `mcp-curator-collection` fails in every run. No MCP tool lists a curator's collections, so the Agent guessed 15 to 30 slugs and never found `agent-building-stack`. +- The MCP server no longer serves curator collections (skilld-dev/skilld.dev#548), so the `mcp-curator-collection` case was removed. It failed in every run: the Agent guessed 15 to 30 slugs and never found `agent-building-stack`. - `mcp-provenance-safety` fails its criteria in every run. Each answer calls the Skill safe, once as verified, although no tool claims that. `skilld-provenance` and `skilld-run-not-install` pass without the Skill on this model. `skilld --help` is enough for them.