diff --git a/docs/evals/writing-2026-10-09/README.md b/docs/evals/writing-2026-10-09/README.md new file mode 100644 index 00000000..5ec755c1 --- /dev/null +++ b/docs/evals/writing-2026-10-09/README.md @@ -0,0 +1,63 @@ +# Writing evaluation, 9 October 2026 + +Six Skills rewrote the same three generated Markdown documents. +OpenCode 1.18.32 used `zai-coding-plan/glm-5.3` for every successful run. +Each Skill ran once. The baseline ran once without Skill instructions. +This run compares writing instructions, not Skill discovery or a stable quality ranking. + +[Recording evidence](record.json) contains exact commits, source hashes, timings, and transcript hashes. +[Seed prompt](seed-prompt.txt) and [rewrite prompt](rewrite-prompt.txt) preserve the shared inputs. +Each directory contains the unedited Markdown output and raw OpenCode events. +The demo manifest stores these same documents for the existing reader. + +## Results + +The no-Skill baseline already removes the seed's obvious padded wording. +All six Skills preserve the exact code block, URLs, and inline identifiers. +The seed and baseline also pass those exact material checks. +Those checks do not establish complete factual preservation or writing quality. + +Word counts below include headings and exclude fenced code. Shorter does not mean better. + +| Run | Article words | README words | PR words | +| --- | ---: | ---: | ---: | +| seed | 217 | 123 | 131 | +| baseline | 157 | 88 | 109 | +| humanizer | 174 | 77 | 94 | +| stop-slop | 148 | 81 | 103 | +| no-ai-slop | 114 | 88 | 77 | +| anti-slop | 158 | 80 | 102 | +| deslop-writing | 133 | 67 | 84 | +| im-not-a-fly | 132 | 79 | 86 | + +## Meaning changes found during agent review + +These observations compare outputs with the supplied fictional facts. +They are agent assessments, not a human review or an automated quality score. + +| Run | Observation | +| --- | --- | +| Baseline | Keeps the app limitations, package behavior, existing category field, and PR exclusions. Adds a generic closing summary. | +| Humanizer | Changes the article's “can be more useful” into “is more useful.” Adds “The rest of the interface is unchanged” to the PR. | +| Stop Slop | Changes “can be more useful” into “serves you better.” Keeps the PR's existing-field detail and exclusions. | +| No AI Slop | Keeps the article's qualified comparison. Omits the PR's detail about using the existing category field. | +| Anti Slop | Keeps the article's qualified comparison. Adds “The rest of the interface is unchanged” to the PR. | +| Deslop Writing | Adds an unsupported requirement that links only become a reading list after sorting. Strengthens the final comparison. Omits the existing-field detail and adds an unchanged-interface claim. | +| Brundlefly im-not-a-fly | Keeps the qualified comparison, existing-field detail, and PR exclusions. Adds a generic causal claim about structure in the article's opening. | + +The PR brief says the change maintains simplicity. It does not establish that every other interface element is unchanged. +All README outputs retain case sensitivity, original ordering, the empty-category behavior, and the unchanged input array. +They also retain the limits on fetching URLs and storing links. +All PR outputs retain bookmark behavior, reload reset, and the search and saved-preference exclusions. + +## Limits + +The topic is fictional. The runner provides the facts and all Markdown source files inside each Skill directory. +The runner disables tools, so Skills cannot inspect external sources or execute scripts. +The model receives Skill instructions inline, including bundled Markdown references. +The baseline uses the same task and original documents without those instructions. +The run uses no temperature override or repeat trials. +The run uses published Brundlefly commit `67d1cf8e3543133a4d73b8a857ff8fc988717956`. +It does not exercise the local checkout's uncommitted `write-human` files. +OpenCode Go rejected an earlier seed attempt because its subscription was inactive. +No output from that attempt enters this comparison. diff --git a/docs/evals/writing-2026-10-09/anti-slop/README.md b/docs/evals/writing-2026-10-09/anti-slop/README.md new file mode 100644 index 00000000..1f3fd64f --- /dev/null +++ b/docs/evals/writing-2026-10-09/anti-slop/README.md @@ -0,0 +1,22 @@ +# Margin + +Margin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category. + +## Getting started + +Import `filterLinks` and pass your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links. + +```ts +import { filterLinks } from 'margin' + +const links = [ + { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' }, + { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' }, +] + +const cssLinks = filterLinks(links, 'CSS') +``` + +## Important considerations + +Category matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is solely a filtering function. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/anti-slop/events.jsonl b/docs/evals/writing-2026-10-09/anti-slop/events.jsonl new file mode 100644 index 00000000..e5aab828 --- /dev/null +++ b/docs/evals/writing-2026-10-09/anti-slop/events.jsonl @@ -0,0 +1,3 @@ +{"type":"step_start","timestamp":1791534032576,"sessionID":"ses_ee0402829ffeR2jsMQdLoZ2TWQ","part":{"id":"prt_11fbfe6ac0016QIcEO7CKHEcAx","messageID":"msg_11fbfd9a0001fkOXl80vY7w7g9","sessionID":"ses_ee0402829ffeR2jsMQdLoZ2TWQ","type":"step-start"}} +{"type":"text","timestamp":1791534097152,"sessionID":"ses_ee0402829ffeR2jsMQdLoZ2TWQ","part":{"id":"prt_11fc0cae1001RW21tMecZ15OgP","messageID":"msg_11fbfd9a0001fkOXl80vY7w7g9","sessionID":"ses_ee0402829ffeR2jsMQdLoZ2TWQ","type":"text","text":"{\"reading-list.md\": \"# Keeping a reading list that works for you\\n\\nA reading list can help you stay informed. With a bit of care, a collection of links becomes a resource you keep learning from.\\n\\n## A simple approach\\n\\nA reading list doesn't need to be complicated. Our fictional reading-list app, Margin, stores a title, URL, and category for each link. You can filter by category and bookmark items you want to return to. Everything stays in your browser. There's no account or cloud sync, which keeps things simple.\\n\\n## Building better habits\\n\\nA weekly review helps you keep a growing collection of articles under control. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. The habit pays off over time.\\n\\nThe goal is a reading list that makes the most of your time. A small, focused list can be more useful than an extensive collection you never open.\", \"README.md\": \"# Margin\\n\\nMargin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category.\\n\\n## Getting started\\n\\nImport `filterLinks` and pass your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Important considerations\\n\\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is solely a filtering function.\", \"pr.md\": \"# Add category filtering to the reading list\\n\\nThis pull request adds category filtering to our fictional reading-list app, Margin. It uses the existing category field, so users can view only the articles in a chosen category.\\n\\nPreviously, the list displayed every saved link regardless of the selected category. Now, selecting a category displays only matching links. Selecting All displays the complete list.\\n\\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\\n\\nThe rest of the interface is unchanged.\"}","time":{"start":1791534090977,"end":1791534097135}}} +{"type":"step_finish","timestamp":1791534097152,"sessionID":"ses_ee0402829ffeR2jsMQdLoZ2TWQ","part":{"id":"prt_11fc0e2f2001U7EpJO9qM90M6Y","reason":"stop","messageID":"msg_11fbfd9a0001fkOXl80vY7w7g9","sessionID":"ses_ee0402829ffeR2jsMQdLoZ2TWQ","type":"step-finish","tokens":{"total":7949,"input":3083,"output":530,"reasoning":4336,"cache":{"write":0,"read":0}},"cost":0}} diff --git a/docs/evals/writing-2026-10-09/anti-slop/pr.md b/docs/evals/writing-2026-10-09/anti-slop/pr.md new file mode 100644 index 00000000..9d804646 --- /dev/null +++ b/docs/evals/writing-2026-10-09/anti-slop/pr.md @@ -0,0 +1,9 @@ +# Add category filtering to the reading list + +This pull request adds category filtering to our fictional reading-list app, Margin. It uses the existing category field, so users can view only the articles in a chosen category. + +Previously, the list displayed every saved link regardless of the selected category. Now, selecting a category displays only matching links. Selecting All displays the complete list. + +Filtering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change. + +The rest of the interface is unchanged. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/anti-slop/reading-list.md b/docs/evals/writing-2026-10-09/anti-slop/reading-list.md new file mode 100644 index 00000000..bddd3020 --- /dev/null +++ b/docs/evals/writing-2026-10-09/anti-slop/reading-list.md @@ -0,0 +1,13 @@ +# Keeping a reading list that works for you + +A reading list can help you stay informed. With a bit of care, a collection of links becomes a resource you keep learning from. + +## A simple approach + +A reading list doesn't need to be complicated. Our fictional reading-list app, Margin, stores a title, URL, and category for each link. You can filter by category and bookmark items you want to return to. Everything stays in your browser. There's no account or cloud sync, which keeps things simple. + +## Building better habits + +A weekly review helps you keep a growing collection of articles under control. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. The habit pays off over time. + +The goal is a reading list that makes the most of your time. A small, focused list can be more useful than an extensive collection you never open. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/baseline/README.md b/docs/evals/writing-2026-10-09/baseline/README.md new file mode 100644 index 00000000..2f9cb211 --- /dev/null +++ b/docs/evals/writing-2026-10-09/baseline/README.md @@ -0,0 +1,22 @@ +# Margin + +Margin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category, so developers can build reading list features. + +## Getting started + +Import `filterLinks` and pass in your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links. + +```ts +import { filterLinks } from 'margin' + +const links = [ + { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' }, + { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' }, +] + +const cssLinks = filterLinks(links, 'CSS') +``` + +## Important considerations + +Category matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is solely a filtering function. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/baseline/events.jsonl b/docs/evals/writing-2026-10-09/baseline/events.jsonl new file mode 100644 index 00000000..11f37c61 --- /dev/null +++ b/docs/evals/writing-2026-10-09/baseline/events.jsonl @@ -0,0 +1,3 @@ +{"type":"step_start","timestamp":1791533726252,"sessionID":"ses_ee044d486ffeNIZpDo6MgsJ0YR","part":{"id":"prt_11fbb3a1c001z6pxEEMmFUxJ4B","messageID":"msg_11fbb2cfb001xwUOa8LSk1j7Cc","sessionID":"ses_ee044d486ffeNIZpDo6MgsJ0YR","type":"step-start"}} +{"type":"text","timestamp":1791533763754,"sessionID":"ses_ee044d486ffeNIZpDo6MgsJ0YR","part":{"id":"prt_11fbbbf3a001ccR1HK8snd1XYI","messageID":"msg_11fbb2cfb001xwUOa8LSk1j7Cc","sessionID":"ses_ee044d486ffeNIZpDo6MgsJ0YR","type":"text","text":"{\n \"reading-list.md\": \"# Keeping a reading list that works for you\\n\\nA reading list helps you stay informed, but only if you can keep up with it. With some care, a collection of links can become a resource you actually learn from.\\n\\n## A simple setup\\n\\nMargin, our fictional reading-list app, keeps things simple. It stores a title, URL, and category for each link. You can filter by category and bookmark items you want to come back to. Everything stays in your browser, so there is no account or cloud sync.\\n\\n## Building better habits\\n\\nA weekly review helps you keep the collection under control. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. That habit pays off over time.\\n\\nThe goal is a reading list that makes good use of your time. A small, focused list can be more useful than an extensive collection you never open.\",\n \"README.md\": \"# Margin\\n\\nMargin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category, so developers can build reading list features.\\n\\n## Getting started\\n\\nImport `filterLinks` and pass in your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Important considerations\\n\\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is solely a filtering function.\",\n \"pr.md\": \"# Add category filtering to the reading list\\n\\nThis pull request adds category filtering to Margin, our fictional reading-list app. It uses the existing category field, so users can narrow their saved articles to a single category.\\n\\nPreviously, the list displayed every saved link regardless of the selected category. Now, selecting a category displays only matching links. Selecting All displays the complete list.\\n\\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\\n\\nThe result is a more focused reading experience, while keeping the existing interface simple.\"\n}","time":{"start":1791533760314,"end":1791533763740}}} +{"type":"step_finish","timestamp":1791533763754,"sessionID":"ses_ee044d486ffeNIZpDo6MgsJ0YR","part":{"id":"prt_11fbbcc9f001taC7GxtX7HNPDy","reason":"stop","messageID":"msg_11fbb2cfb001xwUOa8LSk1j7Cc","sessionID":"ses_ee044d486ffeNIZpDo6MgsJ0YR","type":"step-finish","tokens":{"total":4430,"input":918,"output":548,"reasoning":2964,"cache":{"write":0,"read":0}},"cost":0}} diff --git a/docs/evals/writing-2026-10-09/baseline/pr.md b/docs/evals/writing-2026-10-09/baseline/pr.md new file mode 100644 index 00000000..161ee154 --- /dev/null +++ b/docs/evals/writing-2026-10-09/baseline/pr.md @@ -0,0 +1,9 @@ +# Add category filtering to the reading list + +This pull request adds category filtering to Margin, our fictional reading-list app. It uses the existing category field, so users can narrow their saved articles to a single category. + +Previously, the list displayed every saved link regardless of the selected category. Now, selecting a category displays only matching links. Selecting All displays the complete list. + +Filtering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change. + +The result is a more focused reading experience, while keeping the existing interface simple. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/baseline/reading-list.md b/docs/evals/writing-2026-10-09/baseline/reading-list.md new file mode 100644 index 00000000..5a311468 --- /dev/null +++ b/docs/evals/writing-2026-10-09/baseline/reading-list.md @@ -0,0 +1,13 @@ +# Keeping a reading list that works for you + +A reading list helps you stay informed, but only if you can keep up with it. With some care, a collection of links can become a resource you actually learn from. + +## A simple setup + +Margin, our fictional reading-list app, keeps things simple. It stores a title, URL, and category for each link. You can filter by category and bookmark items you want to come back to. Everything stays in your browser, so there is no account or cloud sync. + +## Building better habits + +A weekly review helps you keep the collection under control. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. That habit pays off over time. + +The goal is a reading list that makes good use of your time. A small, focused list can be more useful than an extensive collection you never open. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/deslop-writing/README.md b/docs/evals/writing-2026-10-09/deslop-writing/README.md new file mode 100644 index 00000000..c9727c0e --- /dev/null +++ b/docs/evals/writing-2026-10-09/deslop-writing/README.md @@ -0,0 +1,22 @@ +# Margin + +Margin is a fictional TypeScript package for organizing reading-list links by category. + +## Getting started + +Import `filterLinks`, then pass it your links and a category. The function returns matching links in their original order. An empty category returns all links. + +```ts +import { filterLinks } from 'margin' + +const links = [ + { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' }, + { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' }, +] + +const cssLinks = filterLinks(links, 'CSS') +``` + +## Behavior and limits + +Category matching is case-sensitive. The function doesn't modify the input array. Margin doesn't fetch URLs or store links; it only filters. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/deslop-writing/events.jsonl b/docs/evals/writing-2026-10-09/deslop-writing/events.jsonl new file mode 100644 index 00000000..69a0f168 --- /dev/null +++ b/docs/evals/writing-2026-10-09/deslop-writing/events.jsonl @@ -0,0 +1,3 @@ +{"type":"step_start","timestamp":1791534105335,"sessionID":"ses_ee03f0d99ffewFhu7xI89LQ4IC","part":{"id":"prt_11fc102e8001YMv6JLZ7U6KnVh","messageID":"msg_11fc0f3d2001Av0sgJQiEogB7Z","sessionID":"ses_ee03f0d99ffewFhu7xI89LQ4IC","type":"step-start"}} +{"type":"text","timestamp":1791534175212,"sessionID":"ses_ee03f0d99ffewFhu7xI89LQ4IC","part":{"id":"prt_11fc20677001HEM0vjoQHarsOF","messageID":"msg_11fc0f3d2001Av0sgJQiEogB7Z","sessionID":"ses_ee03f0d99ffewFhu7xI89LQ4IC","type":"text","text":"{\"reading-list.md\": \"# Keep a reading list that works\\n\\nA pile of saved links only becomes a reading list once you can sort it. The links you keep should be the ones you'll actually return to, and structure is what makes that possible.\\n\\n## What Margin does\\n\\nMargin, our fictional reading-list app, stores three things for each link: a title, URL, and category. You can filter by category and bookmark items you want to come back to. Everything stays in your browser; there's no account and no cloud sync, which keeps things simple.\\n\\n## A weekly review\\n\\nGo through the list once a week. Remove the links you no longer want to read, and keep the ones tied to a project you're working on. A small, focused list beats a big collection you never open.\", \"README.md\": \"# Margin\\n\\nMargin is a fictional TypeScript package for organizing reading-list links by category.\\n\\n## Getting started\\n\\nImport `filterLinks`, then pass it your links and a category. The function returns matching links in their original order. An empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Behavior and limits\\n\\nCategory matching is case-sensitive. The function doesn't modify the input array. Margin doesn't fetch URLs or store links; it only filters.\", \"pr.md\": \"# Add category filtering to the reading list\\n\\nThis PR adds category filtering to Margin, our fictional reading-list app. The list previously showed every saved link regardless of the selected category. Selecting a category now shows only matching links; selecting All shows the complete list.\\n\\nFiltering preserves the original order of links and doesn't change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are out of scope for this change, and the rest of the interface is unchanged.\"}","time":{"start":1791534171767,"end":1791534175197}}} +{"type":"step_finish","timestamp":1791534175212,"sessionID":"ses_ee03f0d99ffewFhu7xI89LQ4IC","part":{"id":"prt_11fc213e1001IhQ7t6fFHtu04a","reason":"stop","messageID":"msg_11fc0f3d2001Av0sgJQiEogB7Z","sessionID":"ses_ee03f0d99ffewFhu7xI89LQ4IC","type":"step-finish","tokens":{"total":14880,"input":10170,"output":466,"reasoning":4244,"cache":{"write":0,"read":0}},"cost":0}} diff --git a/docs/evals/writing-2026-10-09/deslop-writing/pr.md b/docs/evals/writing-2026-10-09/deslop-writing/pr.md new file mode 100644 index 00000000..54243ff5 --- /dev/null +++ b/docs/evals/writing-2026-10-09/deslop-writing/pr.md @@ -0,0 +1,5 @@ +# Add category filtering to the reading list + +This PR adds category filtering to Margin, our fictional reading-list app. The list previously showed every saved link regardless of the selected category. Selecting a category now shows only matching links; selecting All shows the complete list. + +Filtering preserves the original order of links and doesn't change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are out of scope for this change, and the rest of the interface is unchanged. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/deslop-writing/reading-list.md b/docs/evals/writing-2026-10-09/deslop-writing/reading-list.md new file mode 100644 index 00000000..7bb9672b --- /dev/null +++ b/docs/evals/writing-2026-10-09/deslop-writing/reading-list.md @@ -0,0 +1,11 @@ +# Keep a reading list that works + +A pile of saved links only becomes a reading list once you can sort it. The links you keep should be the ones you'll actually return to, and structure is what makes that possible. + +## What Margin does + +Margin, our fictional reading-list app, stores three things for each link: a title, URL, and category. You can filter by category and bookmark items you want to come back to. Everything stays in your browser; there's no account and no cloud sync, which keeps things simple. + +## A weekly review + +Go through the list once a week. Remove the links you no longer want to read, and keep the ones tied to a project you're working on. A small, focused list beats a big collection you never open. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/humanizer/README.md b/docs/evals/writing-2026-10-09/humanizer/README.md new file mode 100644 index 00000000..7e1b650a --- /dev/null +++ b/docs/evals/writing-2026-10-09/humanizer/README.md @@ -0,0 +1,22 @@ +# Margin + +Margin is a fictional TypeScript package for managing reading lists. Its API organizes links by category. + +## Getting started + +Import `filterLinks` and pass it your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links. + +```ts +import { filterLinks } from 'margin' + +const links = [ + { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' }, + { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' }, +] + +const cssLinks = filterLinks(links, 'CSS') +``` + +## Notes + +Category matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links. It is only a filtering function. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/humanizer/events.jsonl b/docs/evals/writing-2026-10-09/humanizer/events.jsonl new file mode 100644 index 00000000..90fd9625 --- /dev/null +++ b/docs/evals/writing-2026-10-09/humanizer/events.jsonl @@ -0,0 +1,3 @@ +{"type":"step_start","timestamp":1791533772172,"sessionID":"ses_ee0442597ffe4n6EjTU8sDt9xm","part":{"id":"prt_11fbbed75001jT5wS7hjhkKrhy","messageID":"msg_11fbbdbfa001XQg3xtvmqjS4KT","sessionID":"ses_ee0442597ffe4n6EjTU8sDt9xm","type":"step-start"}} +{"type":"text","timestamp":1791533902894,"sessionID":"ses_ee0442597ffe4n6EjTU8sDt9xm","part":{"id":"prt_11fbdd6e7001dxiArHqBJDIx4v","messageID":"msg_11fbbdbfa001XQg3xtvmqjS4KT","sessionID":"ses_ee0442597ffe4n6EjTU8sDt9xm","type":"text","text":"{\"reading-list.md\": \"# Keeping a reading list that works for you\\n\\nKeeping a reading list is a good way to stay informed, as long as the list doesn't get out of hand. With some care, a simple collection of links can become a useful resource for learning.\\n\\n## How Margin works\\n\\nA reading list doesn't need to be complicated. Margin, our fictional reading-list app, keeps it that way. It stores a title, URL, and category for each link. You can filter by category and bookmark items you want to come back to. Everything stays in your browser, and there's no account or cloud sync to worry about.\\n\\n## Building better habits\\n\\nA weekly review helps you keep up with the collection. Remove links you no longer want to read, and keep the ones that connect to a project you are working on. The list stays short enough to use.\\n\\nThe goal is a list that makes good use of your reading time. A small, focused list is more useful than an extensive collection you never open.\", \"README.md\": \"# Margin\\n\\nMargin is a fictional TypeScript package for managing reading lists. Its API organizes links by category.\\n\\n## Getting started\\n\\nImport `filterLinks` and pass it your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Notes\\n\\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links. It is only a filtering function.\", \"pr.md\": \"# Add category filtering to the reading list\\n\\nThis pull request adds category filtering to Margin, our fictional reading-list app, using the category field each link already has.\\n\\nPreviously, the list displayed every saved link regardless of the selected category. Selecting a category now displays only matching links. Selecting All displays the complete list.\\n\\nFiltering preserves the original order of the links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\\n\\nThe rest of the interface is unchanged.\"}","time":{"start":1791533897447,"end":1791533902876}}} +{"type":"step_finish","timestamp":1791533902894,"sessionID":"ses_ee0442597ffe4n6EjTU8sDt9xm","part":{"id":"prt_11fbdec20001iNLkeStFJfJW39","reason":"stop","messageID":"msg_11fbbdbfa001XQg3xtvmqjS4KT","sessionID":"ses_ee0442597ffe4n6EjTU8sDt9xm","type":"step-finish","tokens":{"total":14933,"input":8130,"output":537,"reasoning":6266,"cache":{"write":0,"read":0}},"cost":0}} diff --git a/docs/evals/writing-2026-10-09/humanizer/pr.md b/docs/evals/writing-2026-10-09/humanizer/pr.md new file mode 100644 index 00000000..f3de711d --- /dev/null +++ b/docs/evals/writing-2026-10-09/humanizer/pr.md @@ -0,0 +1,9 @@ +# Add category filtering to the reading list + +This pull request adds category filtering to Margin, our fictional reading-list app, using the category field each link already has. + +Previously, the list displayed every saved link regardless of the selected category. Selecting a category now displays only matching links. Selecting All displays the complete list. + +Filtering preserves the original order of the links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change. + +The rest of the interface is unchanged. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/humanizer/reading-list.md b/docs/evals/writing-2026-10-09/humanizer/reading-list.md new file mode 100644 index 00000000..62bc1eef --- /dev/null +++ b/docs/evals/writing-2026-10-09/humanizer/reading-list.md @@ -0,0 +1,13 @@ +# Keeping a reading list that works for you + +Keeping a reading list is a good way to stay informed, as long as the list doesn't get out of hand. With some care, a simple collection of links can become a useful resource for learning. + +## How Margin works + +A reading list doesn't need to be complicated. Margin, our fictional reading-list app, keeps it that way. It stores a title, URL, and category for each link. You can filter by category and bookmark items you want to come back to. Everything stays in your browser, and there's no account or cloud sync to worry about. + +## Building better habits + +A weekly review helps you keep up with the collection. Remove links you no longer want to read, and keep the ones that connect to a project you are working on. The list stays short enough to use. + +The goal is a list that makes good use of your reading time. A small, focused list is more useful than an extensive collection you never open. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/im-not-a-fly/README.md b/docs/evals/writing-2026-10-09/im-not-a-fly/README.md new file mode 100644 index 00000000..4187574e --- /dev/null +++ b/docs/evals/writing-2026-10-09/im-not-a-fly/README.md @@ -0,0 +1,22 @@ +# Margin + +Margin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category. + +## Getting started + +Import `filterLinks` and pass it your links and the category you want. The function returns matching links in their original order. An empty category returns all links. + +```ts +import { filterLinks } from 'margin' + +const links = [ + { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' }, + { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' }, +] + +const cssLinks = filterLinks(links, 'CSS') +``` + +## Notes + +Category matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is only a filtering function. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/im-not-a-fly/events.jsonl b/docs/evals/writing-2026-10-09/im-not-a-fly/events.jsonl new file mode 100644 index 00000000..6c31da02 --- /dev/null +++ b/docs/evals/writing-2026-10-09/im-not-a-fly/events.jsonl @@ -0,0 +1,3 @@ +{"type":"step_start","timestamp":1791534193206,"sessionID":"ses_ee03dcb7affe3xAqig7OaXEMKC","part":{"id":"prt_11fc25a24001DKHkWVKrQRy7mC","messageID":"msg_11fc23649001GeDseXfKn7cYKO","sessionID":"ses_ee03dcb7affe3xAqig7OaXEMKC","type":"step-start"}} +{"type":"text","timestamp":1791534249141,"sessionID":"ses_ee03dcb7affe3xAqig7OaXEMKC","part":{"id":"prt_11fc322fc001gN1ClFe5yny85H","messageID":"msg_11fc23649001GeDseXfKn7cYKO","sessionID":"ses_ee03dcb7affe3xAqig7OaXEMKC","type":"text","text":"{\n \"reading-list.md\": \"# Keeping a reading list that works for you\\n\\nA pile of saved links is not the same as a useful reading list. A little structure turns one into the other, and into something you actually learn from.\\n\\n## A simple setup\\n\\nMargin, our fictional reading-list app, keeps it simple. For each link, Margin stores a title, URL, and category. You can filter by category and bookmark items you want to come back to. Everything stays in your browser. There is no account and no cloud sync.\\n\\n## A weekly review\\n\\nEach week, look over the list. Remove links you no longer want to read. Keep the ones that connect to a project you are working on.\\n\\nA small, focused list can be more useful than an extensive collection you never open.\",\n \"README.md\": \"# Margin\\n\\nMargin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category.\\n\\n## Getting started\\n\\nImport `filterLinks` and pass it your links and the category you want. The function returns matching links in their original order. An empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Notes\\n\\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is only a filtering function.\",\n \"pr.md\": \"# Add category filtering to the reading list\\n\\nThis pull request adds category filtering to Margin, our fictional reading-list app, using the existing category field on each link.\\n\\nPreviously, the list displayed every saved link regardless of the selected category. Selecting a category now displays only matching links. Selecting All displays the complete list.\\n\\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\"\n}","time":{"start":1791534244604,"end":1791534249128}}} +{"type":"step_finish","timestamp":1791534249141,"sessionID":"ses_ee03dcb7affe3xAqig7OaXEMKC","part":{"id":"prt_11fc334ab001QQ6UBYflb1HD0k","reason":"stop","messageID":"msg_11fc23649001GeDseXfKn7cYKO","sessionID":"ses_ee03dcb7affe3xAqig7OaXEMKC","type":"step-finish","tokens":{"total":12516,"input":8962,"output":484,"reasoning":3070,"cache":{"write":0,"read":0}},"cost":0}} diff --git a/docs/evals/writing-2026-10-09/im-not-a-fly/pr.md b/docs/evals/writing-2026-10-09/im-not-a-fly/pr.md new file mode 100644 index 00000000..9cc3fb7c --- /dev/null +++ b/docs/evals/writing-2026-10-09/im-not-a-fly/pr.md @@ -0,0 +1,7 @@ +# Add category filtering to the reading list + +This pull request adds category filtering to Margin, our fictional reading-list app, using the existing category field on each link. + +Previously, the list displayed every saved link regardless of the selected category. Selecting a category now displays only matching links. Selecting All displays the complete list. + +Filtering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/im-not-a-fly/reading-list.md b/docs/evals/writing-2026-10-09/im-not-a-fly/reading-list.md new file mode 100644 index 00000000..6e24a2cd --- /dev/null +++ b/docs/evals/writing-2026-10-09/im-not-a-fly/reading-list.md @@ -0,0 +1,13 @@ +# Keeping a reading list that works for you + +A pile of saved links is not the same as a useful reading list. A little structure turns one into the other, and into something you actually learn from. + +## A simple setup + +Margin, our fictional reading-list app, keeps it simple. For each link, Margin stores a title, URL, and category. You can filter by category and bookmark items you want to come back to. Everything stays in your browser. There is no account and no cloud sync. + +## A weekly review + +Each week, look over the list. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. + +A small, focused list can be more useful than an extensive collection you never open. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/no-ai-slop/README.md b/docs/evals/writing-2026-10-09/no-ai-slop/README.md new file mode 100644 index 00000000..04e2a386 --- /dev/null +++ b/docs/evals/writing-2026-10-09/no-ai-slop/README.md @@ -0,0 +1,24 @@ +# Margin + +Margin is a fictional TypeScript package for managing reading lists. It filters links by category so you can organize a reading list in your own app. + +## Getting started + +Import `filterLinks` and pass it your links and the category you want: + +```ts +import { filterLinks } from 'margin' + +const links = [ + { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' }, + { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' }, +] + +const cssLinks = filterLinks(links, 'CSS') +``` + +The function returns matching links in their original order. Passing an empty category returns all links. + +## Notes + +Category matching is case-sensitive. `filterLinks` does not modify the input array. Margin does not fetch URLs or store links; it only filters the array you pass in. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/no-ai-slop/events.jsonl b/docs/evals/writing-2026-10-09/no-ai-slop/events.jsonl new file mode 100644 index 00000000..df113803 --- /dev/null +++ b/docs/evals/writing-2026-10-09/no-ai-slop/events.jsonl @@ -0,0 +1,3 @@ +{"type":"step_start","timestamp":1791533975987,"sessionID":"ses_ee04104b4ffebHX1472Y6DiOBe","part":{"id":"prt_11fbf0987001DIrK0gsDLVxgbB","messageID":"msg_11fbefcbc001V5BPs8B6mo1B74","sessionID":"ses_ee04104b4ffebHX1472Y6DiOBe","type":"step-start"}} +{"type":"text","timestamp":1791534025725,"sessionID":"ses_ee04104b4ffebHX1472Y6DiOBe","part":{"id":"prt_11fbfbe3d00120dKbN9DQHs4fn","messageID":"msg_11fbefcbc001V5BPs8B6mo1B74","sessionID":"ses_ee04104b4ffebHX1472Y6DiOBe","type":"text","text":"```json\n{\n \"reading-list.md\": \"# Keeping a reading list that works for you\\n\\nA reading list doesn't need to be complicated. Margin, our fictional reading-list app, keeps yours simple, and a weekly review keeps it useful.\\n\\n## A simple approach\\n\\nMargin stores a title, URL, and category for each link. You can filter by category and bookmark items you want to return to. Everything stays in your browser. There's no account or cloud sync.\\n\\n## Building better habits\\n\\nReview the list once a week. Remove links you no longer want to read. Keep the ones that connect to a project you're working on. A small, focused list can be more useful than an extensive collection you never open.\",\n \"README.md\": \"# Margin\\n\\nMargin is a fictional TypeScript package for managing reading lists. It filters links by category so you can organize a reading list in your own app.\\n\\n## Getting started\\n\\nImport `filterLinks` and pass it your links and the category you want:\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\nThe function returns matching links in their original order. Passing an empty category returns all links.\\n\\n## Notes\\n\\nCategory matching is case-sensitive. `filterLinks` does not modify the input array. Margin does not fetch URLs or store links; it only filters the array you pass in.\",\n \"pr.md\": \"# Add category filtering to the reading list\\n\\nThis PR adds category filtering to Margin, our fictional reading-list app. Selecting a category now displays only matching links; selecting All displays the complete list. Previously, the list showed every saved link regardless of the selected category.\\n\\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are out of scope for this change.\"\n}\n```","time":{"start":1791534022205,"end":1791534025712}}} +{"type":"step_finish","timestamp":1791534025725,"sessionID":"ses_ee04104b4ffebHX1472Y6DiOBe","part":{"id":"prt_11fbfcbf3001JRjd6dzcHBw3wW","reason":"stop","messageID":"msg_11fbefcbc001V5BPs8B6mo1B74","sessionID":"ses_ee04104b4ffebHX1472Y6DiOBe","type":"step-finish","tokens":{"total":7814,"input":4134,"output":469,"reasoning":3211,"cache":{"write":0,"read":0}},"cost":0}} diff --git a/docs/evals/writing-2026-10-09/no-ai-slop/pr.md b/docs/evals/writing-2026-10-09/no-ai-slop/pr.md new file mode 100644 index 00000000..0a68075d --- /dev/null +++ b/docs/evals/writing-2026-10-09/no-ai-slop/pr.md @@ -0,0 +1,5 @@ +# Add category filtering to the reading list + +This PR adds category filtering to Margin, our fictional reading-list app. Selecting a category now displays only matching links; selecting All displays the complete list. Previously, the list showed every saved link regardless of the selected category. + +Filtering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are out of scope for this change. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/no-ai-slop/reading-list.md b/docs/evals/writing-2026-10-09/no-ai-slop/reading-list.md new file mode 100644 index 00000000..d25e2f30 --- /dev/null +++ b/docs/evals/writing-2026-10-09/no-ai-slop/reading-list.md @@ -0,0 +1,11 @@ +# Keeping a reading list that works for you + +A reading list doesn't need to be complicated. Margin, our fictional reading-list app, keeps yours simple, and a weekly review keeps it useful. + +## A simple approach + +Margin stores a title, URL, and category for each link. You can filter by category and bookmark items you want to return to. Everything stays in your browser. There's no account or cloud sync. + +## Building better habits + +Review the list once a week. Remove links you no longer want to read. Keep the ones that connect to a project you're working on. A small, focused list can be more useful than an extensive collection you never open. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/record.json b/docs/evals/writing-2026-10-09/record.json new file mode 100644 index 00000000..aedf0513 --- /dev/null +++ b/docs/evals/writing-2026-10-09/record.json @@ -0,0 +1,365 @@ +{ + "model": "zai-coding-plan/glm-5.3", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "seedChecks": [ + { + "file": "reading-list.md", + "missing": [] + }, + { + "file": "README.md", + "missing": [] + }, + { + "file": "pr.md", + "missing": [] + } + ], + "baselineChecks": [ + { + "file": "reading-list.md", + "missing": [] + }, + { + "file": "README.md", + "missing": [] + }, + { + "file": "pr.md", + "missing": [] + } + ], + "runs": [ + { + "label": "seed", + "model": "zai-coding-plan/glm-5.3", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "startedAt": "2026-10-09T08:14:14.142Z", + "completedAt": "2026-10-09T08:15:21.554Z", + "promptSha256": "457e023d0b14ef9b085a884a05f6e17808d1848a4a334e15cb5559c636dc9491", + "eventsSha256": "1dc365aaf46608d58a36b38ff8b8793cfd7b8ea060ef4d5dab4fb5706b03ffae", + "eventsPath": "seed/events.jsonl" + }, + { + "label": "baseline", + "model": "zai-coding-plan/glm-5.3", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "startedAt": "2026-10-09T08:15:21.555Z", + "completedAt": "2026-10-09T08:16:03.819Z", + "promptSha256": "a14a3a178af0aabc21714a4315e5391cb94182968bd1e7955b5d7d78660359b6", + "eventsSha256": "0dc48b0cf1acec348f7dcfc68f9b6480804c6a1228b5275862928bca6da6e95a", + "eventsPath": "baseline/events.jsonl" + }, + { + "label": "humanizer", + "model": "zai-coding-plan/glm-5.3", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "startedAt": "2026-10-09T08:16:06.087Z", + "completedAt": "2026-10-09T08:18:22.985Z", + "promptSha256": "3bd3e8eb7baff15ce49fb4f818c8f472c1931ff304ff50f71fbb1677f7809bfc", + "eventsSha256": "657768673f69ef01b34d53ca27bcfd36a49993aedc5b312f2a97f00d08be5fe5", + "eventsPath": "humanizer/events.jsonl" + }, + { + "label": "stop-slop", + "model": "zai-coding-plan/glm-5.3", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "startedAt": "2026-10-09T08:18:27.990Z", + "completedAt": "2026-10-09T08:19:28.619Z", + "promptSha256": "40fab1b14614ee3a068a6ac3ed38b29323a78f333cd75fe67d5546b4f20aac9c", + "eventsSha256": "f23c423b78b3c045ef94fe5ee9af356b648147483bcdb2ada65dac0590bb2d3e", + "eventsPath": "stop-slop/events.jsonl" + }, + { + "label": "no-ai-slop", + "model": "zai-coding-plan/glm-5.3", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "startedAt": "2026-10-09T08:19:31.322Z", + "completedAt": "2026-10-09T08:20:25.781Z", + "promptSha256": "1819e2699b97fe68b21d49e43e219acce8e9d84dd1b1cb17874bf8a79927d0aa", + "eventsSha256": "8398d47ff552186d604784667ac70a337cfd392ed16f8c43ba6602666d07c4d7", + "eventsPath": "no-ai-slop/events.jsonl" + }, + { + "label": "anti-slop", + "model": "zai-coding-plan/glm-5.3", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "startedAt": "2026-10-09T08:20:27.568Z", + "completedAt": "2026-10-09T08:21:37.214Z", + "promptSha256": "8e1531cee7e4e30f76958c4cc1e7c5c82401eb4847b1a926a8287d8e5ed617d0", + "eventsSha256": "249eda3083603ecb9493e0df3c3c60213247e0a479ddbf5d98ee498b87e39649", + "eventsPath": "anti-slop/events.jsonl" + }, + { + "label": "deslop-writing", + "model": "zai-coding-plan/glm-5.3", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "startedAt": "2026-10-09T08:21:40.186Z", + "completedAt": "2026-10-09T08:22:55.266Z", + "promptSha256": "bea7ac8f469dbfefe3502dc9b715f792df62f1377a989525b46f21ff2cdf98f6", + "eventsSha256": "07b4492b4d30c49033b2a65d7dcf0efb53400030dc70b2fc6d3d58bcce3f2e32", + "eventsPath": "deslop-writing/events.jsonl" + }, + { + "label": "im-not-a-fly", + "model": "zai-coding-plan/glm-5.3", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "startedAt": "2026-10-09T08:23:02.192Z", + "completedAt": "2026-10-09T08:24:09.201Z", + "promptSha256": "768bbe4b05652e05679ca3e148da030c0c11e55c59c000807084e1c9fde61da3", + "eventsSha256": "786e8be557df966ce0268ca781b20f8bcb3bc0e319fc618fe21756a3cf421aad", + "eventsPath": "im-not-a-fly/events.jsonl" + } + ], + "skills": [ + { + "owner": "blader", + "repo": "humanizer", + "name": "humanizer", + "source": { + "sourceCommit": "225a6f39ac85f76ee48dbad772ea4abe4ed6c9d8", + "sourceUrl": "https://github.com/blader/humanizer/blob/225a6f39ac85f76ee48dbad772ea4abe4ed6c9d8/SKILL.md", + "skillPath": "SKILL.md" + }, + "sourceFiles": [ + { + "path": "SKILL.md", + "sha256": "0612f1dfb1672b0ea9b97e139bf1f06cabe98d8b27424fe8ff01e1fb4cc99cad" + } + ], + "materialChecks": [ + { + "file": "reading-list.md", + "missing": [] + }, + { + "file": "README.md", + "missing": [] + }, + { + "file": "pr.md", + "missing": [] + } + ] + }, + { + "owner": "hardikpandya", + "repo": "stop-slop", + "name": "stop-slop", + "source": { + "sourceCommit": "8da1f030185bdfe8471220585162991eaeb970e9", + "sourceUrl": "https://github.com/hardikpandya/stop-slop/blob/8da1f030185bdfe8471220585162991eaeb970e9/SKILL.md", + "skillPath": "SKILL.md" + }, + "sourceFiles": [ + { + "path": "SKILL.md", + "sha256": "7432a1d9ebdd42b27666da8458af252edf549f723fb14bcd4791425103930310" + }, + { + "path": "references/examples.md", + "sha256": "49c592899eb4457f95d0767e15e3143569e2cef50f2927c3174e62432984cd3d" + }, + { + "path": "references/phrases.md", + "sha256": "4ca1b7e97123b5c67eff3dc97e5e977332e0b24021265c1553888a93ec33fa93" + }, + { + "path": "references/structures.md", + "sha256": "ffbf1316c7fb5aaf552de005609a64a9639b546ecd76bfb0e482864d5bbe6f58" + } + ], + "materialChecks": [ + { + "file": "reading-list.md", + "missing": [] + }, + { + "file": "README.md", + "missing": [] + }, + { + "file": "pr.md", + "missing": [] + } + ] + }, + { + "owner": "petergyang", + "repo": "no-ai-slop", + "name": "no-ai-slop", + "source": { + "sourceCommit": "000650b156983f5159695b441477f4e63b25dc85", + "sourceUrl": "https://github.com/petergyang/no-ai-slop/blob/000650b156983f5159695b441477f4e63b25dc85/skills/no-ai-slop/SKILL.md", + "skillPath": "skills/no-ai-slop/SKILL.md" + }, + "sourceFiles": [ + { + "path": "skills/no-ai-slop/SKILL.md", + "sha256": "992b365f51a2f62cf4c1c5ed22049a9ee551dfea677550c1a72b371c0ceadd62" + }, + { + "path": "skills/no-ai-slop/eval.md", + "sha256": "8ad8d83ed1abe7fc054ad74966bfca64fc182bc71f59d701b24b962c39d11ad7" + } + ], + "materialChecks": [ + { + "file": "reading-list.md", + "missing": [] + }, + { + "file": "README.md", + "missing": [] + }, + { + "file": "pr.md", + "missing": [] + } + ] + }, + { + "owner": "elithrar", + "repo": "dotfiles", + "name": "anti-slop", + "source": { + "sourceCommit": "26e8a4b2f26f6de2a22eecb02a6463235d6adf08", + "sourceUrl": "https://github.com/elithrar/dotfiles/blob/26e8a4b2f26f6de2a22eecb02a6463235d6adf08/.agents/skills/anti-slop/SKILL.md", + "skillPath": ".agents/skills/anti-slop/SKILL.md" + }, + "sourceFiles": [ + { + "path": ".agents/skills/anti-slop/SKILL.md", + "sha256": "45f2c2e9347822f18022883672b49ab48b3d8cf221b63c2eeef19f4972e7e0ea" + }, + { + "path": ".agents/skills/anti-slop/references/tells.md", + "sha256": "dc20b3f75436963b43d3179ed3f040294887b6c3f08edefc5840d73f498dc6ec" + } + ], + "materialChecks": [ + { + "file": "reading-list.md", + "missing": [] + }, + { + "file": "README.md", + "missing": [] + }, + { + "file": "pr.md", + "missing": [] + } + ] + }, + { + "owner": "kmaida", + "repo": "deslop-skills", + "name": "deslop-writing", + "source": { + "sourceCommit": "70237e1ae0dfbf7bc9ab5e6e3de52c092586ca9b", + "sourceUrl": "https://github.com/kmaida/deslop-skills/blob/70237e1ae0dfbf7bc9ab5e6e3de52c092586ca9b/deslop-writing/SKILL.md", + "skillPath": "deslop-writing/SKILL.md" + }, + "sourceFiles": [ + { + "path": "deslop-writing/SKILL.md", + "sha256": "9f405867f6e17f203754f29d10e937a0aa5294f3396794ed04e17093bca4edf7" + }, + { + "path": "deslop-writing/references/apple-style-tech-docs.md", + "sha256": "64e8c830582a08bb7df6321c3986e1eb787f07a27454af71be11b228a6058b04" + }, + { + "path": "deslop-writing/references/banned-words.md", + "sha256": "74f915454cb4d17640ee7daf02c31d414978a8b3333141af60b483ebd4c227d9" + }, + { + "path": "deslop-writing/tests/clean-sample.md", + "sha256": "578405d153552b6ee11680acce7695410dd180672e101c55d3176678e7a7fc4d" + }, + { + "path": "deslop-writing/tests/slop-sample.md", + "sha256": "e15d3e47567c341367d2666d452d5bb3c4fbf410db69076fc5b7c8ca8b079fa9" + } + ], + "materialChecks": [ + { + "file": "reading-list.md", + "missing": [] + }, + { + "file": "README.md", + "missing": [] + }, + { + "file": "pr.md", + "missing": [] + } + ] + }, + { + "owner": "harlan-zw", + "repo": "brundlefly", + "name": "im-not-a-fly", + "source": { + "sourceCommit": "67d1cf8e3543133a4d73b8a857ff8fc988717956", + "sourceUrl": "https://github.com/harlan-zw/brundlefly/blob/67d1cf8e3543133a4d73b8a857ff8fc988717956/skills/im-not-a-fly/SKILL.md", + "skillPath": "skills/im-not-a-fly/SKILL.md" + }, + "sourceFiles": [ + { + "path": "skills/im-not-a-fly/SKILL.md", + "sha256": "5aed63bd8b0d090b942ccc7a3d31cda8ee8f8ddcf31b6db3c7c8ca6b478bbbb6" + }, + { + "path": "skills/im-not-a-fly/references/accessible-writing.md", + "sha256": "447be0e2586f752d804d1061e6b19eb717b17a1f8d251ccbde44d14a62e83a7b" + }, + { + "path": "skills/im-not-a-fly/references/blocks/claim-fidelity.md", + "sha256": "0c27dca805003965a26a411a65c1a78bccaad42a046d1112e38045cb575ee69f" + }, + { + "path": "skills/im-not-a-fly/references/blocks/clarity.md", + "sha256": "9051774e8dcd6da29f26fc371a416638f8b21606b089a60f971551bfd627d162" + }, + { + "path": "skills/im-not-a-fly/references/graphite-ai-tells-2026.md", + "sha256": "2fc98fe7bf3b3ca472a78088fceaa0077a21932d5d158c291ebd23dbbb76f713" + }, + { + "path": "skills/im-not-a-fly/references/storytelling.md", + "sha256": "242b114d5638bdd7a14d382c69919709526a7bce6445ff5bd034c028c5faf04f" + }, + { + "path": "skills/im-not-a-fly/references/writing-well.md", + "sha256": "5275e6e532575cfc01d853bab223c2924e710f394256145288bef2659c63bc9b" + } + ], + "materialChecks": [ + { + "file": "reading-list.md", + "missing": [] + }, + { + "file": "README.md", + "missing": [] + }, + { + "file": "pr.md", + "missing": [] + } + ] + } + ] +} diff --git a/docs/evals/writing-2026-10-09/rewrite-prompt.txt b/docs/evals/writing-2026-10-09/rewrite-prompt.txt new file mode 100644 index 00000000..6e7aa76b --- /dev/null +++ b/docs/evals/writing-2026-10-09/rewrite-prompt.txt @@ -0,0 +1,4 @@ +Rewrite these Markdown documents to remove AI writing tells and make them clear and natural. Preserve all facts, uncertainty, code, links, identifiers, and useful document structure. Return only the requested documents, without review notes. +Documents: +{"reading-list.md":"# Keeping a reading list that works for you\n\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\n\n## A streamlined approach\n\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\n\n## Building better habits\n\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\n\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.","README.md":"# Margin\n\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\n\n## Getting started\n\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.","pr.md":"# Add category filtering to the reading list\n\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\n\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\n\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\n\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface."} +Return one JSON object with exactly these keys: "reading-list.md", "README.md", "pr.md". Each value is the complete Markdown document as a JSON string. No outer Markdown fence or commentary. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/seed-prompt.txt b/docs/evals/writing-2026-10-09/seed-prompt.txt new file mode 100644 index 00000000..e32c4f5f --- /dev/null +++ b/docs/evals/writing-2026-10-09/seed-prompt.txt @@ -0,0 +1,3 @@ +Draft three deliberately generic, AI-sounding Markdown documents about the fictional Margin reading-list app and package. Use padded transitions, vague praise, and uniform prose. Do not invent facts, first-person experiences, capabilities, tests, measurements, or sources. Keep each document under 500 words. Preserve every fact, code block, URL, identifier, and heading from this supplied brief. The app and package are separate examples, do not merge their capabilities. +{"reading-list.md":"# Keeping a reading list that works for you\n\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. By leveraging a thoughtful approach, you can transform a collection of links into a valuable resource for learning and growth.\n\n## A streamlined approach\n\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, stores a title, URL, and category for each link. You can filter by category and bookmark items you want to return to. Everything stays in your browser. There is no account or cloud sync.\n\n## Building better habits\n\nFurthermore, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on.\n\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. A small, focused list can be more useful than an extensive collection you never open.","README.md":"# Margin\n\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\n\n## Getting started\n\nTo begin leveraging Margin, import `filterLinks` and pass your links and the desired category. The function returns matching links in their original order. Passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nIt's important to note that category matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links. It is solely a filtering function, making it a valuable addition to your toolkit.","pr.md":"# Add category filtering to the reading list\n\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\n\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. Selecting All displays the complete list.\n\nIt's worth noting that filtering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\n\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface."} +Return one JSON object with exactly these keys: "reading-list.md", "README.md", "pr.md". Each value is the complete Markdown document as a JSON string. No outer Markdown fence or commentary. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/seed/README.md b/docs/evals/writing-2026-10-09/seed/README.md new file mode 100644 index 00000000..300e5a75 --- /dev/null +++ b/docs/evals/writing-2026-10-09/seed/README.md @@ -0,0 +1,22 @@ +# Margin + +Margin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences. + +## Getting started + +To begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links. + +```ts +import { filterLinks } from 'margin' + +const links = [ + { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' }, + { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' }, +] + +const cssLinks = filterLinks(links, 'CSS') +``` + +## Important considerations + +It's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/seed/events.jsonl b/docs/evals/writing-2026-10-09/seed/events.jsonl new file mode 100644 index 00000000..bcc3dc86 --- /dev/null +++ b/docs/evals/writing-2026-10-09/seed/events.jsonl @@ -0,0 +1,3 @@ +{"type":"step_start","timestamp":1791533661337,"sessionID":"ses_ee045dac4ffeUQwRKEyBeRmhrl","part":{"id":"prt_11fba3c87001hOkJ1tla7Vpqs0","messageID":"msg_11fba2702001nWgnwBijhCl8mz","sessionID":"ses_ee045dac4ffeUQwRKEyBeRmhrl","type":"step-start"}} +{"type":"text","timestamp":1791533721491,"sessionID":"ses_ee045dac4ffeUQwRKEyBeRmhrl","part":{"id":"prt_11fbb10b3001XX2qI8FpMc8xgI","messageID":"msg_11fba2702001nWgnwBijhCl8mz","sessionID":"ses_ee045dac4ffeUQwRKEyBeRmhrl","type":"text","text":"{\"reading-list.md\":\"# Keeping a reading list that works for you\\n\\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\\n\\n## A streamlined approach\\n\\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\\n\\n## Building better habits\\n\\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\\n\\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.\",\"README.md\":\"# Margin\\n\\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\\n\\n## Getting started\\n\\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Important considerations\\n\\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.\",\"pr.md\":\"# Add category filtering to the reading list\\n\\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\\n\\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\\n\\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\\n\\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.\"}","time":{"start":1791533715635,"end":1791533721477}}} +{"type":"step_finish","timestamp":1791533721492,"sessionID":"ses_ee045dac4ffeUQwRKEyBeRmhrl","part":{"id":"prt_11fbb27880015g6vn5VV012UHs","reason":"stop","messageID":"msg_11fba2702001nWgnwBijhCl8mz","sessionID":"ses_ee045dac4ffeUQwRKEyBeRmhrl","type":"step-finish","tokens":{"total":5861,"input":883,"output":684,"reasoning":4294,"cache":{"write":0,"read":0}},"cost":0}} diff --git a/docs/evals/writing-2026-10-09/seed/pr.md b/docs/evals/writing-2026-10-09/seed/pr.md new file mode 100644 index 00000000..bcc3787c --- /dev/null +++ b/docs/evals/writing-2026-10-09/seed/pr.md @@ -0,0 +1,9 @@ +# Add category filtering to the reading list + +This pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency. + +Previously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list. + +Furthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change. + +Ultimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/seed/reading-list.md b/docs/evals/writing-2026-10-09/seed/reading-list.md new file mode 100644 index 00000000..e0154edb --- /dev/null +++ b/docs/evals/writing-2026-10-09/seed/reading-list.md @@ -0,0 +1,13 @@ +# Keeping a reading list that works for you + +In today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth. + +## A streamlined approach + +It's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward. + +## Building better habits + +Moreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time. + +Ultimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/stop-slop/README.md b/docs/evals/writing-2026-10-09/stop-slop/README.md new file mode 100644 index 00000000..fbe65101 --- /dev/null +++ b/docs/evals/writing-2026-10-09/stop-slop/README.md @@ -0,0 +1,22 @@ +# Margin + +Margin is a fictional TypeScript package for managing reading lists. It provides a small API for organizing links by category. + +## Getting started + +Import `filterLinks` and pass it your links and the category you want. The function returns matching links in their original order. An empty category returns all links. + +```ts +import { filterLinks } from 'margin' + +const links = [ + { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' }, + { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' }, +] + +const cssLinks = filterLinks(links, 'CSS') +``` + +## Notes + +Category matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links. Margin filters links and does nothing else. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/stop-slop/events.jsonl b/docs/evals/writing-2026-10-09/stop-slop/events.jsonl new file mode 100644 index 00000000..27dda7ed --- /dev/null +++ b/docs/evals/writing-2026-10-09/stop-slop/events.jsonl @@ -0,0 +1,3 @@ +{"type":"step_start","timestamp":1791533913549,"sessionID":"ses_ee041fb7dffe97SFwKRIIN7JJ5","part":{"id":"prt_11fbe15bc001Slp9fiiJOQYLZg","messageID":"msg_11fbe061d0013tr8bXOYDJ6v03","sessionID":"ses_ee041fb7dffe97SFwKRIIN7JJ5","type":"step-start"}} +{"type":"text","timestamp":1791533968572,"sessionID":"ses_ee041fb7dffe97SFwKRIIN7JJ5","part":{"id":"prt_11fbedc71001DDlubc95Xaxndd","messageID":"msg_11fbe061d0013tr8bXOYDJ6v03","sessionID":"ses_ee041fb7dffe97SFwKRIIN7JJ5","type":"text","text":"{\"reading-list.md\": \"# Keeping a reading list that works for you\\n\\nManaging a reading list is part of staying informed. With a bit of care, you can turn a pile of links into a resource you learn from.\\n\\n## A simple setup\\n\\nA reading list doesn't need to be complicated. Our fictional reading-list app, Margin, keeps it simple. Margin stores a title, URL, and category for each link. You can filter by category and bookmark items you want to return to. Everything stays in your browser. There is no account and no cloud sync.\\n\\n## Building better habits\\n\\nA weekly review helps you manage a growing list of articles. Remove links you no longer want to read. Keep the ones tied to a project you're working on.\\n\\nAim for a reading rhythm that fits your time. A short, focused list serves you better than a long one you never open.\", \"README.md\": \"# Margin\\n\\nMargin is a fictional TypeScript package for managing reading lists. It provides a small API for organizing links by category.\\n\\n## Getting started\\n\\nImport `filterLinks` and pass it your links and the category you want. The function returns matching links in their original order. An empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Notes\\n\\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links. Margin filters links and does nothing else.\", \"pr.md\": \"# Add category filtering to the reading list\\n\\nThis PR adds category filtering to Margin, our fictional reading-list app. It uses the existing category field, so selecting a category narrows the saved articles to that one.\\n\\nPreviously, the list displayed every saved link regardless of the selected category. Now selecting a category displays only matching links. Selecting All displays the complete list.\\n\\nFiltering preserves the original order of links and leaves bookmarks unchanged. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\\n\\nThe rest of the interface stays as simple as before.\"}","time":{"start":1791533964401,"end":1791533968557}}} +{"type":"step_finish","timestamp":1791533968572,"sessionID":"ses_ee041fb7dffe97SFwKRIIN7JJ5","part":{"id":"prt_11fbeecb1001Mr4QOd38q35BMV","reason":"stop","messageID":"msg_11fbe061d0013tr8bXOYDJ6v03","sessionID":"ses_ee041fb7dffe97SFwKRIIN7JJ5","type":"step-finish","tokens":{"total":8140,"input":4205,"output":522,"reasoning":3413,"cache":{"write":0,"read":0}},"cost":0}} diff --git a/docs/evals/writing-2026-10-09/stop-slop/pr.md b/docs/evals/writing-2026-10-09/stop-slop/pr.md new file mode 100644 index 00000000..c0c1b023 --- /dev/null +++ b/docs/evals/writing-2026-10-09/stop-slop/pr.md @@ -0,0 +1,9 @@ +# Add category filtering to the reading list + +This PR adds category filtering to Margin, our fictional reading-list app. It uses the existing category field, so selecting a category narrows the saved articles to that one. + +Previously, the list displayed every saved link regardless of the selected category. Now selecting a category displays only matching links. Selecting All displays the complete list. + +Filtering preserves the original order of links and leaves bookmarks unchanged. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change. + +The rest of the interface stays as simple as before. \ No newline at end of file diff --git a/docs/evals/writing-2026-10-09/stop-slop/reading-list.md b/docs/evals/writing-2026-10-09/stop-slop/reading-list.md new file mode 100644 index 00000000..0035f530 --- /dev/null +++ b/docs/evals/writing-2026-10-09/stop-slop/reading-list.md @@ -0,0 +1,13 @@ +# Keeping a reading list that works for you + +Managing a reading list is part of staying informed. With a bit of care, you can turn a pile of links into a resource you learn from. + +## A simple setup + +A reading list doesn't need to be complicated. Our fictional reading-list app, Margin, keeps it simple. Margin stores a title, URL, and category for each link. You can filter by category and bookmark items you want to return to. Everything stays in your browser. There is no account and no cloud sync. + +## Building better habits + +A weekly review helps you manage a growing list of articles. Remove links you no longer want to read. Keep the ones tied to a project you're working on. + +Aim for a reading rhythm that fits your time. A short, focused list serves you better than a long one you never open. \ No newline at end of file diff --git a/docs/runbooks/writing-evals.md b/docs/runbooks/writing-evals.md new file mode 100644 index 00000000..a1863583 --- /dev/null +++ b/docs/runbooks/writing-evals.md @@ -0,0 +1,44 @@ +# Writing evaluations + +Run from a task worktree: + +```sh +pnpm exec tsx scripts/eval-writing.ts --out ~/scratch/writing-evals-YYYYMMDD --model zai-coding-plan/glm-5.3 +``` + +The runner generates three AI-sounding Markdown documents about the fictional Margin app and package. +The existing document fixtures supply the facts, code, and links. +It generates one rewrite without a Skill, then one rewrite for each known writing Skill. +Every rewrite receives the same original documents and task prompt. +Each OpenCode run uses a fresh project, isolated home, and disabled tools. +The runner disables global Skills, external plugins, sharing, and automatic updates. +Provider credentials pass through memory and never enter the output files. +[OpenCode permissions](https://opencode.ai/docs/permissions/) define the tool controls. + +Each Skill comes from its registry source commit. +The snapshot includes its Markdown files and bundled Markdown references. +The runner injects these instructions into the prompt, rather than testing Skill discovery. +It uses the published Brundlefly Skill. It never reads the local package's uncommitted changes. + +The output directory keeps prompts, raw event transcripts, final Markdown, and source hashes. +`results.json` stores original, baseline, and Skill output for each file. +The runner resumes completed runs only when their prompt, model, and OpenCode version match. +If a run returns malformed JSON, an error, or tool activity, it fails. + +## Read the results + +Exact material checks find removed or changed code blocks, inline identifiers, commands, and URLs. +These checks do not prove factual completeness, natural writing, or authorship. +Read every document for invented claims, lost conditions, and changes to meaning. +Compare the baseline with the Skill output before making claims about improvement. +One run per Skill is a demonstration, not a stable quality ranking. +All outputs remain machine-generated and must keep that disclosure. + +## Publish a reviewed run + +Review `results.json` and the Markdown files first. +Keep seed and baseline identical across the six records. +Use the recorded model, OpenCode version, source commit, date, and exact rewrite prompt. +Add the documents to the existing demo manifest with `makes: writing` and an empty `shots` array. +Keep provenance in the evaluation record and open a pull request. +The normal production workflow publishes the approved records after merge. diff --git a/eslint.config.mjs b/eslint.config.mjs index 9ac60888..bf19fe74 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -3,8 +3,8 @@ import harlanzw from 'eslint-plugin-harlanzw' export default antfu({ // skilld sync writes .skills from skilld-dev/skilld. CI checks it with check:skills. - // Immutable research exports retain their recorded byte hashes. - ignores: ['.skills/**', 'public/research/**/measurements.json'], + // Immutable research exports and recorded evaluation documents keep their original bytes. + ignores: ['.skills/**', 'public/research/**/measurements.json', 'docs/evals/writing-*/*/{reading-list,README,pr}.md'], rules: { 'vue/no-useless-v-bind': 'off', 'vue/attribute-hyphenation': 'off', diff --git a/layers/registry/server/data/skill-demos.json b/layers/registry/server/data/skill-demos.json index 88c833c2..3d1cfdd9 100644 --- a/layers/registry/server/data/skill-demos.json +++ b/layers/registry/server/data/skill-demos.json @@ -1584,6 +1584,264 @@ "alt": "Phone screenshot of the page the Agent built for: Make a 7-slide speaker-led talk deck on how Thimblecrest, a fictional orchard co-op, doubled its apple harvest by rentin" } ] + }, + { + "owner": "blader", + "repo": "humanizer", + "name": "humanizer", + "makes": "writing", + "sourceUrl": "https://github.com/blader/humanizer/blob/225a6f39ac85f76ee48dbad772ea4abe4ed6c9d8/SKILL.md", + "skillCommit": "225a6f39ac85f76ee48dbad772ea4abe4ed6c9d8", + "prompt": "Rewrite these Markdown documents to remove AI writing tells and make them clear and natural. Preserve all facts, uncertainty, code, links, identifiers, and useful document structure. Return only the requested documents, without review notes.\nDocuments:\n{\"reading-list.md\":\"# Keeping a reading list that works for you\\n\\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\\n\\n## A streamlined approach\\n\\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\\n\\n## Building better habits\\n\\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\\n\\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.\",\"README.md\":\"# Margin\\n\\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\\n\\n## Getting started\\n\\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Important considerations\\n\\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.\",\"pr.md\":\"# Add category filtering to the reading list\\n\\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\\n\\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\\n\\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\\n\\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.\"}\nReturn one JSON object with exactly these keys: \"reading-list.md\", \"README.md\", \"pr.md\". Each value is the complete Markdown document as a JSON string. No outer Markdown fence or commentary.", + "setup": "AI-generated original Markdown and a shared no-Skill rewrite. Three documents about the fictional Margin app and package.", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "model": "zai-coding-plan/glm-5.3", + "recordedAt": "2026-10-09", + "writing": { + "documents": [ + { + "id": "article", + "label": "reading-list.md", + "format": "article", + "original": "# Keeping a reading list that works for you\n\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\n\n## A streamlined approach\n\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\n\n## Building better habits\n\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\n\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.", + "baseline": "# Keeping a reading list that works for you\n\nA reading list helps you stay informed, but only if you can keep up with it. With some care, a collection of links can become a resource you actually learn from.\n\n## A simple setup\n\nMargin, our fictional reading-list app, keeps things simple. It stores a title, URL, and category for each link. You can filter by category and bookmark items you want to come back to. Everything stays in your browser, so there is no account or cloud sync.\n\n## Building better habits\n\nA weekly review helps you keep the collection under control. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. That habit pays off over time.\n\nThe goal is a reading list that makes good use of your time. A small, focused list can be more useful than an extensive collection you never open.", + "output": "# Keeping a reading list that works for you\n\nKeeping a reading list is a good way to stay informed, as long as the list doesn't get out of hand. With some care, a simple collection of links can become a useful resource for learning.\n\n## How Margin works\n\nA reading list doesn't need to be complicated. Margin, our fictional reading-list app, keeps it that way. It stores a title, URL, and category for each link. You can filter by category and bookmark items you want to come back to. Everything stays in your browser, and there's no account or cloud sync to worry about.\n\n## Building better habits\n\nA weekly review helps you keep up with the collection. Remove links you no longer want to read, and keep the ones that connect to a project you are working on. The list stays short enough to use.\n\nThe goal is a list that makes good use of your reading time. A small, focused list is more useful than an extensive collection you never open." + }, + { + "id": "readme", + "label": "README.md", + "format": "readme", + "original": "# Margin\n\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\n\n## Getting started\n\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.", + "baseline": "# Margin\n\nMargin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category, so developers can build reading list features.\n\n## Getting started\n\nImport `filterLinks` and pass in your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is solely a filtering function.", + "output": "# Margin\n\nMargin is a fictional TypeScript package for managing reading lists. Its API organizes links by category.\n\n## Getting started\n\nImport `filterLinks` and pass it your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Notes\n\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links. It is only a filtering function." + }, + { + "id": "pr", + "label": "pr.md", + "format": "pr", + "original": "# Add category filtering to the reading list\n\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\n\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\n\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\n\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.", + "baseline": "# Add category filtering to the reading list\n\nThis pull request adds category filtering to Margin, our fictional reading-list app. It uses the existing category field, so users can narrow their saved articles to a single category.\n\nPreviously, the list displayed every saved link regardless of the selected category. Now, selecting a category displays only matching links. Selecting All displays the complete list.\n\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\n\nThe result is a more focused reading experience, while keeping the existing interface simple.", + "output": "# Add category filtering to the reading list\n\nThis pull request adds category filtering to Margin, our fictional reading-list app, using the category field each link already has.\n\nPreviously, the list displayed every saved link regardless of the selected category. Selecting a category now displays only matching links. Selecting All displays the complete list.\n\nFiltering preserves the original order of the links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\n\nThe rest of the interface is unchanged." + } + ] + }, + "shots": [] + }, + { + "owner": "hardikpandya", + "repo": "stop-slop", + "name": "stop-slop", + "makes": "writing", + "sourceUrl": "https://github.com/hardikpandya/stop-slop/blob/8da1f030185bdfe8471220585162991eaeb970e9/SKILL.md", + "skillCommit": "8da1f030185bdfe8471220585162991eaeb970e9", + "prompt": "Rewrite these Markdown documents to remove AI writing tells and make them clear and natural. Preserve all facts, uncertainty, code, links, identifiers, and useful document structure. Return only the requested documents, without review notes.\nDocuments:\n{\"reading-list.md\":\"# Keeping a reading list that works for you\\n\\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\\n\\n## A streamlined approach\\n\\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\\n\\n## Building better habits\\n\\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\\n\\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.\",\"README.md\":\"# Margin\\n\\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\\n\\n## Getting started\\n\\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Important considerations\\n\\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.\",\"pr.md\":\"# Add category filtering to the reading list\\n\\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\\n\\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\\n\\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\\n\\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.\"}\nReturn one JSON object with exactly these keys: \"reading-list.md\", \"README.md\", \"pr.md\". Each value is the complete Markdown document as a JSON string. No outer Markdown fence or commentary.", + "setup": "AI-generated original Markdown and a shared no-Skill rewrite. Three documents about the fictional Margin app and package.", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "model": "zai-coding-plan/glm-5.3", + "recordedAt": "2026-10-09", + "writing": { + "documents": [ + { + "id": "article", + "label": "reading-list.md", + "format": "article", + "original": "# Keeping a reading list that works for you\n\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\n\n## A streamlined approach\n\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\n\n## Building better habits\n\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\n\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.", + "baseline": "# Keeping a reading list that works for you\n\nA reading list helps you stay informed, but only if you can keep up with it. With some care, a collection of links can become a resource you actually learn from.\n\n## A simple setup\n\nMargin, our fictional reading-list app, keeps things simple. It stores a title, URL, and category for each link. You can filter by category and bookmark items you want to come back to. Everything stays in your browser, so there is no account or cloud sync.\n\n## Building better habits\n\nA weekly review helps you keep the collection under control. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. That habit pays off over time.\n\nThe goal is a reading list that makes good use of your time. A small, focused list can be more useful than an extensive collection you never open.", + "output": "# Keeping a reading list that works for you\n\nManaging a reading list is part of staying informed. With a bit of care, you can turn a pile of links into a resource you learn from.\n\n## A simple setup\n\nA reading list doesn't need to be complicated. Our fictional reading-list app, Margin, keeps it simple. Margin stores a title, URL, and category for each link. You can filter by category and bookmark items you want to return to. Everything stays in your browser. There is no account and no cloud sync.\n\n## Building better habits\n\nA weekly review helps you manage a growing list of articles. Remove links you no longer want to read. Keep the ones tied to a project you're working on.\n\nAim for a reading rhythm that fits your time. A short, focused list serves you better than a long one you never open." + }, + { + "id": "readme", + "label": "README.md", + "format": "readme", + "original": "# Margin\n\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\n\n## Getting started\n\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.", + "baseline": "# Margin\n\nMargin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category, so developers can build reading list features.\n\n## Getting started\n\nImport `filterLinks` and pass in your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is solely a filtering function.", + "output": "# Margin\n\nMargin is a fictional TypeScript package for managing reading lists. It provides a small API for organizing links by category.\n\n## Getting started\n\nImport `filterLinks` and pass it your links and the category you want. The function returns matching links in their original order. An empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Notes\n\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links. Margin filters links and does nothing else." + }, + { + "id": "pr", + "label": "pr.md", + "format": "pr", + "original": "# Add category filtering to the reading list\n\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\n\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\n\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\n\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.", + "baseline": "# Add category filtering to the reading list\n\nThis pull request adds category filtering to Margin, our fictional reading-list app. It uses the existing category field, so users can narrow their saved articles to a single category.\n\nPreviously, the list displayed every saved link regardless of the selected category. Now, selecting a category displays only matching links. Selecting All displays the complete list.\n\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\n\nThe result is a more focused reading experience, while keeping the existing interface simple.", + "output": "# Add category filtering to the reading list\n\nThis PR adds category filtering to Margin, our fictional reading-list app. It uses the existing category field, so selecting a category narrows the saved articles to that one.\n\nPreviously, the list displayed every saved link regardless of the selected category. Now selecting a category displays only matching links. Selecting All displays the complete list.\n\nFiltering preserves the original order of links and leaves bookmarks unchanged. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\n\nThe rest of the interface stays as simple as before." + } + ] + }, + "shots": [] + }, + { + "owner": "petergyang", + "repo": "no-ai-slop", + "name": "no-ai-slop", + "makes": "writing", + "sourceUrl": "https://github.com/petergyang/no-ai-slop/blob/000650b156983f5159695b441477f4e63b25dc85/skills/no-ai-slop/SKILL.md", + "skillCommit": "000650b156983f5159695b441477f4e63b25dc85", + "prompt": "Rewrite these Markdown documents to remove AI writing tells and make them clear and natural. Preserve all facts, uncertainty, code, links, identifiers, and useful document structure. Return only the requested documents, without review notes.\nDocuments:\n{\"reading-list.md\":\"# Keeping a reading list that works for you\\n\\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\\n\\n## A streamlined approach\\n\\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\\n\\n## Building better habits\\n\\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\\n\\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.\",\"README.md\":\"# Margin\\n\\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\\n\\n## Getting started\\n\\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Important considerations\\n\\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.\",\"pr.md\":\"# Add category filtering to the reading list\\n\\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\\n\\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\\n\\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\\n\\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.\"}\nReturn one JSON object with exactly these keys: \"reading-list.md\", \"README.md\", \"pr.md\". Each value is the complete Markdown document as a JSON string. No outer Markdown fence or commentary.", + "setup": "AI-generated original Markdown and a shared no-Skill rewrite. Three documents about the fictional Margin app and package.", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "model": "zai-coding-plan/glm-5.3", + "recordedAt": "2026-10-09", + "writing": { + "documents": [ + { + "id": "article", + "label": "reading-list.md", + "format": "article", + "original": "# Keeping a reading list that works for you\n\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\n\n## A streamlined approach\n\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\n\n## Building better habits\n\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\n\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.", + "baseline": "# Keeping a reading list that works for you\n\nA reading list helps you stay informed, but only if you can keep up with it. With some care, a collection of links can become a resource you actually learn from.\n\n## A simple setup\n\nMargin, our fictional reading-list app, keeps things simple. It stores a title, URL, and category for each link. You can filter by category and bookmark items you want to come back to. Everything stays in your browser, so there is no account or cloud sync.\n\n## Building better habits\n\nA weekly review helps you keep the collection under control. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. That habit pays off over time.\n\nThe goal is a reading list that makes good use of your time. A small, focused list can be more useful than an extensive collection you never open.", + "output": "# Keeping a reading list that works for you\n\nA reading list doesn't need to be complicated. Margin, our fictional reading-list app, keeps yours simple, and a weekly review keeps it useful.\n\n## A simple approach\n\nMargin stores a title, URL, and category for each link. You can filter by category and bookmark items you want to return to. Everything stays in your browser. There's no account or cloud sync.\n\n## Building better habits\n\nReview the list once a week. Remove links you no longer want to read. Keep the ones that connect to a project you're working on. A small, focused list can be more useful than an extensive collection you never open." + }, + { + "id": "readme", + "label": "README.md", + "format": "readme", + "original": "# Margin\n\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\n\n## Getting started\n\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.", + "baseline": "# Margin\n\nMargin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category, so developers can build reading list features.\n\n## Getting started\n\nImport `filterLinks` and pass in your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is solely a filtering function.", + "output": "# Margin\n\nMargin is a fictional TypeScript package for managing reading lists. It filters links by category so you can organize a reading list in your own app.\n\n## Getting started\n\nImport `filterLinks` and pass it your links and the category you want:\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\nThe function returns matching links in their original order. Passing an empty category returns all links.\n\n## Notes\n\nCategory matching is case-sensitive. `filterLinks` does not modify the input array. Margin does not fetch URLs or store links; it only filters the array you pass in." + }, + { + "id": "pr", + "label": "pr.md", + "format": "pr", + "original": "# Add category filtering to the reading list\n\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\n\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\n\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\n\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.", + "baseline": "# Add category filtering to the reading list\n\nThis pull request adds category filtering to Margin, our fictional reading-list app. It uses the existing category field, so users can narrow their saved articles to a single category.\n\nPreviously, the list displayed every saved link regardless of the selected category. Now, selecting a category displays only matching links. Selecting All displays the complete list.\n\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\n\nThe result is a more focused reading experience, while keeping the existing interface simple.", + "output": "# Add category filtering to the reading list\n\nThis PR adds category filtering to Margin, our fictional reading-list app. Selecting a category now displays only matching links; selecting All displays the complete list. Previously, the list showed every saved link regardless of the selected category.\n\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are out of scope for this change." + } + ] + }, + "shots": [] + }, + { + "owner": "elithrar", + "repo": "dotfiles", + "name": "anti-slop", + "makes": "writing", + "sourceUrl": "https://github.com/elithrar/dotfiles/blob/26e8a4b2f26f6de2a22eecb02a6463235d6adf08/.agents/skills/anti-slop/SKILL.md", + "skillCommit": "26e8a4b2f26f6de2a22eecb02a6463235d6adf08", + "prompt": "Rewrite these Markdown documents to remove AI writing tells and make them clear and natural. Preserve all facts, uncertainty, code, links, identifiers, and useful document structure. Return only the requested documents, without review notes.\nDocuments:\n{\"reading-list.md\":\"# Keeping a reading list that works for you\\n\\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\\n\\n## A streamlined approach\\n\\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\\n\\n## Building better habits\\n\\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\\n\\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.\",\"README.md\":\"# Margin\\n\\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\\n\\n## Getting started\\n\\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Important considerations\\n\\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.\",\"pr.md\":\"# Add category filtering to the reading list\\n\\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\\n\\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\\n\\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\\n\\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.\"}\nReturn one JSON object with exactly these keys: \"reading-list.md\", \"README.md\", \"pr.md\". Each value is the complete Markdown document as a JSON string. No outer Markdown fence or commentary.", + "setup": "AI-generated original Markdown and a shared no-Skill rewrite. Three documents about the fictional Margin app and package.", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "model": "zai-coding-plan/glm-5.3", + "recordedAt": "2026-10-09", + "writing": { + "documents": [ + { + "id": "article", + "label": "reading-list.md", + "format": "article", + "original": "# Keeping a reading list that works for you\n\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\n\n## A streamlined approach\n\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\n\n## Building better habits\n\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\n\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.", + "baseline": "# Keeping a reading list that works for you\n\nA reading list helps you stay informed, but only if you can keep up with it. With some care, a collection of links can become a resource you actually learn from.\n\n## A simple setup\n\nMargin, our fictional reading-list app, keeps things simple. It stores a title, URL, and category for each link. You can filter by category and bookmark items you want to come back to. Everything stays in your browser, so there is no account or cloud sync.\n\n## Building better habits\n\nA weekly review helps you keep the collection under control. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. That habit pays off over time.\n\nThe goal is a reading list that makes good use of your time. A small, focused list can be more useful than an extensive collection you never open.", + "output": "# Keeping a reading list that works for you\n\nA reading list can help you stay informed. With a bit of care, a collection of links becomes a resource you keep learning from.\n\n## A simple approach\n\nA reading list doesn't need to be complicated. Our fictional reading-list app, Margin, stores a title, URL, and category for each link. You can filter by category and bookmark items you want to return to. Everything stays in your browser. There's no account or cloud sync, which keeps things simple.\n\n## Building better habits\n\nA weekly review helps you keep a growing collection of articles under control. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. The habit pays off over time.\n\nThe goal is a reading list that makes the most of your time. A small, focused list can be more useful than an extensive collection you never open." + }, + { + "id": "readme", + "label": "README.md", + "format": "readme", + "original": "# Margin\n\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\n\n## Getting started\n\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.", + "baseline": "# Margin\n\nMargin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category, so developers can build reading list features.\n\n## Getting started\n\nImport `filterLinks` and pass in your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is solely a filtering function.", + "output": "# Margin\n\nMargin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category.\n\n## Getting started\n\nImport `filterLinks` and pass your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is solely a filtering function." + }, + { + "id": "pr", + "label": "pr.md", + "format": "pr", + "original": "# Add category filtering to the reading list\n\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\n\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\n\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\n\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.", + "baseline": "# Add category filtering to the reading list\n\nThis pull request adds category filtering to Margin, our fictional reading-list app. It uses the existing category field, so users can narrow their saved articles to a single category.\n\nPreviously, the list displayed every saved link regardless of the selected category. Now, selecting a category displays only matching links. Selecting All displays the complete list.\n\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\n\nThe result is a more focused reading experience, while keeping the existing interface simple.", + "output": "# Add category filtering to the reading list\n\nThis pull request adds category filtering to our fictional reading-list app, Margin. It uses the existing category field, so users can view only the articles in a chosen category.\n\nPreviously, the list displayed every saved link regardless of the selected category. Now, selecting a category displays only matching links. Selecting All displays the complete list.\n\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\n\nThe rest of the interface is unchanged." + } + ] + }, + "shots": [] + }, + { + "owner": "kmaida", + "repo": "deslop-skills", + "name": "deslop-writing", + "makes": "writing", + "sourceUrl": "https://github.com/kmaida/deslop-skills/blob/70237e1ae0dfbf7bc9ab5e6e3de52c092586ca9b/deslop-writing/SKILL.md", + "skillCommit": "70237e1ae0dfbf7bc9ab5e6e3de52c092586ca9b", + "prompt": "Rewrite these Markdown documents to remove AI writing tells and make them clear and natural. Preserve all facts, uncertainty, code, links, identifiers, and useful document structure. Return only the requested documents, without review notes.\nDocuments:\n{\"reading-list.md\":\"# Keeping a reading list that works for you\\n\\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\\n\\n## A streamlined approach\\n\\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\\n\\n## Building better habits\\n\\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\\n\\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.\",\"README.md\":\"# Margin\\n\\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\\n\\n## Getting started\\n\\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Important considerations\\n\\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.\",\"pr.md\":\"# Add category filtering to the reading list\\n\\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\\n\\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\\n\\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\\n\\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.\"}\nReturn one JSON object with exactly these keys: \"reading-list.md\", \"README.md\", \"pr.md\". Each value is the complete Markdown document as a JSON string. No outer Markdown fence or commentary.", + "setup": "AI-generated original Markdown and a shared no-Skill rewrite. Three documents about the fictional Margin app and package.", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "model": "zai-coding-plan/glm-5.3", + "recordedAt": "2026-10-09", + "writing": { + "documents": [ + { + "id": "article", + "label": "reading-list.md", + "format": "article", + "original": "# Keeping a reading list that works for you\n\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\n\n## A streamlined approach\n\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\n\n## Building better habits\n\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\n\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.", + "baseline": "# Keeping a reading list that works for you\n\nA reading list helps you stay informed, but only if you can keep up with it. With some care, a collection of links can become a resource you actually learn from.\n\n## A simple setup\n\nMargin, our fictional reading-list app, keeps things simple. It stores a title, URL, and category for each link. You can filter by category and bookmark items you want to come back to. Everything stays in your browser, so there is no account or cloud sync.\n\n## Building better habits\n\nA weekly review helps you keep the collection under control. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. That habit pays off over time.\n\nThe goal is a reading list that makes good use of your time. A small, focused list can be more useful than an extensive collection you never open.", + "output": "# Keep a reading list that works\n\nA pile of saved links only becomes a reading list once you can sort it. The links you keep should be the ones you'll actually return to, and structure is what makes that possible.\n\n## What Margin does\n\nMargin, our fictional reading-list app, stores three things for each link: a title, URL, and category. You can filter by category and bookmark items you want to come back to. Everything stays in your browser; there's no account and no cloud sync, which keeps things simple.\n\n## A weekly review\n\nGo through the list once a week. Remove the links you no longer want to read, and keep the ones tied to a project you're working on. A small, focused list beats a big collection you never open." + }, + { + "id": "readme", + "label": "README.md", + "format": "readme", + "original": "# Margin\n\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\n\n## Getting started\n\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.", + "baseline": "# Margin\n\nMargin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category, so developers can build reading list features.\n\n## Getting started\n\nImport `filterLinks` and pass in your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is solely a filtering function.", + "output": "# Margin\n\nMargin is a fictional TypeScript package for organizing reading-list links by category.\n\n## Getting started\n\nImport `filterLinks`, then pass it your links and a category. The function returns matching links in their original order. An empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Behavior and limits\n\nCategory matching is case-sensitive. The function doesn't modify the input array. Margin doesn't fetch URLs or store links; it only filters." + }, + { + "id": "pr", + "label": "pr.md", + "format": "pr", + "original": "# Add category filtering to the reading list\n\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\n\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\n\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\n\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.", + "baseline": "# Add category filtering to the reading list\n\nThis pull request adds category filtering to Margin, our fictional reading-list app. It uses the existing category field, so users can narrow their saved articles to a single category.\n\nPreviously, the list displayed every saved link regardless of the selected category. Now, selecting a category displays only matching links. Selecting All displays the complete list.\n\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\n\nThe result is a more focused reading experience, while keeping the existing interface simple.", + "output": "# Add category filtering to the reading list\n\nThis PR adds category filtering to Margin, our fictional reading-list app. The list previously showed every saved link regardless of the selected category. Selecting a category now shows only matching links; selecting All shows the complete list.\n\nFiltering preserves the original order of links and doesn't change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are out of scope for this change, and the rest of the interface is unchanged." + } + ] + }, + "shots": [] + }, + { + "owner": "harlan-zw", + "repo": "brundlefly", + "name": "im-not-a-fly", + "makes": "writing", + "sourceUrl": "https://github.com/harlan-zw/brundlefly/blob/67d1cf8e3543133a4d73b8a857ff8fc988717956/skills/im-not-a-fly/SKILL.md", + "skillCommit": "67d1cf8e3543133a4d73b8a857ff8fc988717956", + "prompt": "Rewrite these Markdown documents to remove AI writing tells and make them clear and natural. Preserve all facts, uncertainty, code, links, identifiers, and useful document structure. Return only the requested documents, without review notes.\nDocuments:\n{\"reading-list.md\":\"# Keeping a reading list that works for you\\n\\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\\n\\n## A streamlined approach\\n\\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\\n\\n## Building better habits\\n\\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\\n\\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.\",\"README.md\":\"# Margin\\n\\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\\n\\n## Getting started\\n\\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\\n\\n```ts\\nimport { filterLinks } from 'margin'\\n\\nconst links = [\\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\\n]\\n\\nconst cssLinks = filterLinks(links, 'CSS')\\n```\\n\\n## Important considerations\\n\\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.\",\"pr.md\":\"# Add category filtering to the reading list\\n\\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\\n\\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\\n\\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\\n\\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.\"}\nReturn one JSON object with exactly these keys: \"reading-list.md\", \"README.md\", \"pr.md\". Each value is the complete Markdown document as a JSON string. No outer Markdown fence or commentary.", + "setup": "AI-generated original Markdown and a shared no-Skill rewrite. Three documents about the fictional Margin app and package.", + "agent": "OpenCode", + "agentVersion": "1.18.32", + "model": "zai-coding-plan/glm-5.3", + "recordedAt": "2026-10-09", + "writing": { + "documents": [ + { + "id": "article", + "label": "reading-list.md", + "format": "article", + "original": "# Keeping a reading list that works for you\n\nIn today's fast-paced digital landscape, managing a reading list has become an increasingly important part of staying informed. Indeed, it goes without saying that by leveraging a thoughtful approach, you can transform a simple collection of links into a valuable resource for learning and growth.\n\n## A streamlined approach\n\nIt's worth noting that a reading list doesn't need to be complicated. Our fictional reading-list app, Margin, offers a streamlined approach that truly delivers. Margin stores a title, URL, and category for each link. Furthermore, you can filter by category and bookmark items you want to return to. Everything stays in your browser. It's important to note that there is no account or cloud sync, which keeps things refreshingly straightforward.\n\n## Building better habits\n\nMoreover, a weekly review can help you navigate the ever-growing collection of articles. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. This simple habit can make a world of difference over time.\n\nUltimately, the goal is to create a seamless reading experience that empowers you to make the most of your time. At the end of the day, a small, focused list can be more useful than an extensive collection you never open.", + "baseline": "# Keeping a reading list that works for you\n\nA reading list helps you stay informed, but only if you can keep up with it. With some care, a collection of links can become a resource you actually learn from.\n\n## A simple setup\n\nMargin, our fictional reading-list app, keeps things simple. It stores a title, URL, and category for each link. You can filter by category and bookmark items you want to come back to. Everything stays in your browser, so there is no account or cloud sync.\n\n## Building better habits\n\nA weekly review helps you keep the collection under control. Remove links you no longer want to read. Keep the ones that connect to a project you are working on. That habit pays off over time.\n\nThe goal is a reading list that makes good use of your time. A small, focused list can be more useful than an extensive collection you never open.", + "output": "# Keeping a reading list that works for you\n\nA pile of saved links is not the same as a useful reading list. A little structure turns one into the other, and into something you actually learn from.\n\n## A simple setup\n\nMargin, our fictional reading-list app, keeps it simple. For each link, Margin stores a title, URL, and category. You can filter by category and bookmark items you want to come back to. Everything stays in your browser. There is no account and no cloud sync.\n\n## A weekly review\n\nEach week, look over the list. Remove links you no longer want to read. Keep the ones that connect to a project you are working on.\n\nA small, focused list can be more useful than an extensive collection you never open." + }, + { + "id": "readme", + "label": "README.md", + "format": "readme", + "original": "# Margin\n\nMargin is a fictional TypeScript package designed to streamline the management of reading lists. It provides a robust yet intuitive API for organizing links by category, enabling developers to create seamless reading experiences.\n\n## Getting started\n\nTo begin leveraging Margin, simply import `filterLinks` and pass your links and the desired category. It's worth noting that the function returns matching links in their original order. Furthermore, passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nIt's important to note that category matching is case-sensitive. Moreover, the function does not modify the input array. It's also worth highlighting that Margin does not fetch URLs or store links. Ultimately, Margin is solely a filtering function, making it a valuable addition to your toolkit.", + "baseline": "# Margin\n\nMargin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category, so developers can build reading list features.\n\n## Getting started\n\nImport `filterLinks` and pass in your links and the category you want. The function returns matching links in their original order. Passing an empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Important considerations\n\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is solely a filtering function.", + "output": "# Margin\n\nMargin is a fictional TypeScript package for managing reading lists. It provides an API for organizing links by category.\n\n## Getting started\n\nImport `filterLinks` and pass it your links and the category you want. The function returns matching links in their original order. An empty category returns all links.\n\n```ts\nimport { filterLinks } from 'margin'\n\nconst links = [\n { title: 'CSS notes', url: 'https://example.com/css', category: 'CSS' },\n { title: 'TypeScript notes', url: 'https://example.com/ts', category: 'TypeScript' },\n]\n\nconst cssLinks = filterLinks(links, 'CSS')\n```\n\n## Notes\n\nCategory matching is case-sensitive. The function does not modify the input array. Margin does not fetch URLs or store links; it is only a filtering function." + }, + { + "id": "pr", + "label": "pr.md", + "format": "pr", + "original": "# Add category filtering to the reading list\n\nThis pull request introduces a streamlined category filtering experience to our fictional reading-list app, Margin. By leveraging the existing category field, users can now navigate their saved articles with greater ease and efficiency.\n\nPreviously, the list displayed every saved link regardless of the selected category. With this enhancement, selecting a category displays only matching links. It's worth noting that selecting All displays the complete list.\n\nFurthermore, filtering preserves the original order of links and does not change bookmarks. It's important to note that the selected category resets when the page reloads. Moreover, search and saved filter preferences are outside the scope of this change.\n\nUltimately, this improvement provides a more focused and intuitive reading experience while maintaining the simplicity of the existing interface.", + "baseline": "# Add category filtering to the reading list\n\nThis pull request adds category filtering to Margin, our fictional reading-list app. It uses the existing category field, so users can narrow their saved articles to a single category.\n\nPreviously, the list displayed every saved link regardless of the selected category. Now, selecting a category displays only matching links. Selecting All displays the complete list.\n\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change.\n\nThe result is a more focused reading experience, while keeping the existing interface simple.", + "output": "# Add category filtering to the reading list\n\nThis pull request adds category filtering to Margin, our fictional reading-list app, using the existing category field on each link.\n\nPreviously, the list displayed every saved link regardless of the selected category. Selecting a category now displays only matching links. Selecting All displays the complete list.\n\nFiltering preserves the original order of links and does not change bookmarks. The selected category resets when the page reloads. Search and saved filter preferences are outside the scope of this change." + } + ] + }, + "shots": [] } ] } diff --git a/scripts/eval-writing.ts b/scripts/eval-writing.ts new file mode 100644 index 00000000..280be6cf --- /dev/null +++ b/scripts/eval-writing.ts @@ -0,0 +1,145 @@ +/** OpenCode writing comparison. Outputs and transcripts stay in --out until reviewed. */ +import type { EvalDocuments } from './lib/writing-eval' +import { execFile } from 'node:child_process' +import { createHash } from 'node:crypto' +import { existsSync } from 'node:fs' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { homedir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { parseArgs, promisify } from 'node:util' +import { z } from 'zod' +import { writingDemoExample, writingDemoSkills } from '../app/utils/writing-demo-example' +import { checkWritingMaterial, documentLabels, evalDocumentsSchema, parseWritingResponse, rewritePrompt } from './lib/writing-eval' + +const exec = promisify(execFile) +const { values } = parseArgs({ options: { out: { type: 'string' }, model: { type: 'string', default: 'zai-coding-plan/glm-5.3' } } }) +if (!values.out) + throw new Error('Pass --out with a new scratch directory.') +const out = resolve(values.out) +const model = values.model! +await mkdir(out, { recursive: true }) +const authPath = join(homedir(), '.local/share/opencode/auth.json') +const auth = existsSync(authPath) ? await readFile(authPath, 'utf8') : null +const configPath = join(homedir(), '.config/opencode/opencode.json') +const provider = existsSync(configPath) ? (JSON.parse(await readFile(configPath, 'utf8')) as { provider?: unknown }).provider : undefined +const version = (await exec('opencode', ['--version'])).stdout.trim() +const config = { + $schema: 'https://opencode.ai/config.json', + ...(provider ? { provider } : {}), + autoupdate: false, + share: 'disabled', + permission: { '*': 'deny' }, + agent: { writing: { description: 'Controlled Markdown writing comparison', mode: 'primary', prompt: 'You edit or draft Markdown using only the supplied material. Follow the requested output format. Do not use tools.', permission: { '*': 'deny' } } }, +} +const hash = (text: string) => createHash('sha256').update(text).digest('hex') +const envelope = '\nReturn one JSON object with exactly these keys: "reading-list.md", "README.md", "pr.md". Each value is the complete Markdown document as a JSON string. No outer Markdown fence or commentary.' +async function run(label: string, prompt: string): Promise { + const dir = join(out, label) + if (existsSync(join(dir, 'output.json'))) { + const savedPrompt = await readFile(join(dir, 'prompt.txt'), 'utf8') + const savedModel = await readFile(join(dir, 'model.txt'), 'utf8') + const savedRun = z.object({ agentVersion: z.string() }).parse(JSON.parse(await readFile(join(dir, 'run.json'), 'utf8'))) + if (savedPrompt !== prompt || savedModel !== model || savedRun.agentVersion !== version) + throw new Error(`Cannot resume ${label}: prompt, model, or OpenCode version changed.`) + return parseWritingResponse(await readFile(join(dir, 'events.jsonl'), 'utf8')) + } + await mkdir(join(dir, 'project'), { recursive: true }) + await writeFile(join(dir, 'prompt.txt'), prompt) + await writeFile(join(dir, 'model.txt'), model) + const env: NodeJS.ProcessEnv = { + PATH: process.env.PATH, + LANG: process.env.LANG, + HOME: dir, + XDG_CONFIG_HOME: join(dir, 'config'), + XDG_DATA_HOME: join(dir, 'data'), + XDG_STATE_HOME: join(dir, 'state'), + OPENCODE_CONFIG_CONTENT: JSON.stringify(config), + ...(auth ? { OPENCODE_AUTH_CONTENT: auth } : {}), + OPENCODE_DISABLE_EXTERNAL_SKILLS: '1', + OPENCODE_DISABLE_CLAUDE_CODE: '1', + OPENCODE_DISABLE_AUTOUPDATE: '1', + OPENCODE_DISABLE_SHARE: '1', + } + console.log(`Running ${label} with ${model}`) + const startedAt = new Date().toISOString() + const execution = exec('opencode', ['run', '--pure', '--format', 'json', '--agent', 'writing', '--dir', join(dir, 'project'), '--model', model, prompt], { env, timeout: 600_000, maxBuffer: 32 * 1024 * 1024 }) + execution.child.stdin?.end() + const result = await execution.catch(async (error: { stdout?: string, stderr?: string }) => { + await writeFile(join(dir, 'events.jsonl'), error.stdout ?? '') + await writeFile(join(dir, 'stderr.log'), error.stderr ?? '') + throw error + }) + await writeFile(join(dir, 'events.jsonl'), result.stdout) + await writeFile(join(dir, 'stderr.log'), result.stderr) + const documents = parseWritingResponse(result.stdout) + await writeFile(join(dir, 'output.json'), `${JSON.stringify(documents, null, 2)}\n`) + await writeFile(join(dir, 'run.json'), `${JSON.stringify({ model, agent: 'OpenCode', agentVersion: version, startedAt, completedAt: new Date().toISOString(), promptSha256: hash(prompt) }, null, 2)}\n`) + for (const file of documentLabels) + await writeFile(join(dir, file), documents[file]) + return documents +} + +const sourceSchema = z.object({ sourceCommit: z.string().regex(/^[a-f0-9]{40}$/), sourceUrl: z.string().url(), skillPath: z.string().min(1) }) +async function github(path: string): Promise { + return JSON.parse((await exec('agent-gh', ['api', path], { maxBuffer: 32 * 1024 * 1024 })).stdout) +} +async function snapshot(skill: typeof writingDemoSkills[number]) { + const key = `${skill.owner}/${skill.repo}/${skill.name}` + const file = join(out, `${skill.name}-source.json`) + if (existsSync(file)) + return z.object({ key: z.literal(key), source: sourceSchema, files: z.array(z.object({ path: z.string(), text: z.string(), sha256: z.string() })) }).parse(JSON.parse(await readFile(file, 'utf8'))) + const response = await fetch(`https://skilld.dev/api/v1/skills/${key}`) + if (!response.ok) + throw new Error(`${key}: source lookup answered ${response.status}.`) + const source = sourceSchema.parse(await response.json()) + const tree = z.object({ truncated: z.boolean(), tree: z.array(z.object({ path: z.string(), type: z.string() })) }).parse(await github(`repos/${skill.owner}/${skill.repo}/git/trees/${source.sourceCommit}?recursive=1`)) + if (tree.truncated) + throw new Error(`${key}: source tree is incomplete.`) + const directory = dirname(source.skillPath) + const paths = tree.tree.filter(entry => entry.type === 'blob' && entry.path.endsWith('.md') && (directory === '.' ? entry.path === source.skillPath || entry.path.startsWith('references/') : entry.path.startsWith(`${directory}/`))).map(entry => entry.path) + if (!paths.includes(source.skillPath)) + throw new Error(`${key}: SKILL.md is missing from the pinned tree.`) + const files = [] + for (const path of paths) { + const blob = z.object({ encoding: z.literal('base64'), content: z.string() }).parse(await github(`repos/${skill.owner}/${skill.repo}/contents/${path}?ref=${source.sourceCommit}`)) + const text = Buffer.from(blob.content, 'base64').toString('utf8') + files.push({ path, text, sha256: hash(text) }) + } + const saved = { key, source, files } + await writeFile(file, `${JSON.stringify(saved, null, 2)}\n`) + return saved +} + +const facts = evalDocumentsSchema.parse(Object.fromEntries(writingDemoExample.documents.map(doc => [doc.label, doc.original]))) +const original = await run('seed', `Draft three deliberately generic, AI-sounding Markdown documents about the fictional Margin reading-list app and package. Use padded transitions, vague praise, and uniform prose. Do not invent facts, first-person experiences, capabilities, tests, measurements, or sources. Keep each document under 500 words. Preserve every fact, code block, URL, identifier, and heading from this supplied brief. The app and package are separate examples, do not merge their capabilities.\n${JSON.stringify(facts)}${envelope}`) +const baseline = await run('baseline', `${rewritePrompt}\nDocuments:\n${JSON.stringify(original)}${envelope}`) +const results = [] +for (const skill of writingDemoSkills) { + const pinned = await snapshot(skill) + const instructions = pinned.files.map(file => `File: ${file.path}\n${file.text}`).join('\n\n') + const output = await run(skill.name, `Apply this Skill and its bundled references to the task below. These supplied files are the complete source snapshot. Use only the supplied fictional facts; do not claim external verification. Follow the requested document-only output format even if the Skill asks for review notes.\n\n${instructions}\n\n\n${rewritePrompt}\nDocuments:\n${JSON.stringify(original)}${envelope}`) + results.push({ ...skill, source: pinned.source, sourceFiles: pinned.files.map(({ path, sha256 }) => ({ path, sha256 })), materialChecks: checkWritingMaterial(original, output), writing: { documents: writingDemoExample.documents.map(doc => ({ id: doc.id, label: doc.label, format: doc.format, original: original[doc.label as keyof EvalDocuments], baseline: baseline[doc.label as keyof EvalDocuments], output: output[doc.label as keyof EvalDocuments] })) } }) + await writeFile(join(out, 'results.json'), `${JSON.stringify({ model, agent: 'OpenCode', agentVersion: version, rewritePrompt, seedChecks: checkWritingMaterial(facts, original), baselineChecks: checkWritingMaterial(original, baseline), results }, null, 2)}\n`) +} +const demos = [] +for (const result of results) { + const run = z.object({ completedAt: z.string() }).parse(JSON.parse(await readFile(join(out, result.name, 'run.json'), 'utf8'))) + demos.push({ + owner: result.owner, + repo: result.repo, + name: result.name, + makes: 'writing', + sourceUrl: result.source.sourceUrl, + skillCommit: result.source.sourceCommit, + prompt: `${rewritePrompt}\nDocuments:\n${JSON.stringify(original)}${envelope}`, + setup: 'AI-generated original Markdown and a shared no-Skill rewrite. Three documents about the fictional Margin app and package.', + agent: 'OpenCode', + agentVersion: version, + model, + recordedAt: run.completedAt.slice(0, 10), + writing: result.writing, + shots: [], + }) +} +await writeFile(join(out, 'demos.json'), `${JSON.stringify({ demos }, null, 2)}\n`) +console.log(`Recorded ${results.length} Skills. Review ${join(out, 'results.json')} before publishing.`) diff --git a/scripts/lib/writing-eval.ts b/scripts/lib/writing-eval.ts new file mode 100644 index 00000000..242574c0 --- /dev/null +++ b/scripts/lib/writing-eval.ts @@ -0,0 +1,32 @@ +import { z } from 'zod' + +export const documentLabels = ['reading-list.md', 'README.md', 'pr.md'] as const +export const rewritePrompt = 'Rewrite these Markdown documents to remove AI writing tells and make them clear and natural. Preserve all facts, uncertainty, code, links, identifiers, and useful document structure. Return only the requested documents, without review notes.' +const markdown = z.string().refine(text => text.trim().length > 0, 'A document needs Markdown.') +export const evalDocumentsSchema = z.object({ + 'reading-list.md': markdown, + 'README.md': markdown, + 'pr.md': markdown, +}).strict() +export type EvalDocuments = z.infer + +/** Refuse malformed responses and tool activity rather than silently recording a partial run. */ +export function parseWritingResponse(events: string): EvalDocuments { + const rows = events.trim().split('\n').filter(Boolean).map(line => JSON.parse(line) as { type: string, part?: { text?: string }, error?: unknown }) + const failure = rows.find(row => row.type === 'error' || row.type === 'tool_use') + if (failure) + throw new Error(`OpenCode returned ${failure.type}: ${JSON.stringify(failure.error ?? failure.part)}`) + const text = rows.filter(row => row.type === 'text').map(row => row.part?.text ?? '').join('\n').trim() + return evalDocumentsSchema.parse(JSON.parse(text.replace(/^```json\s*\n/, '').replace(/\n```$/, ''))) +} + +/** Exact material is checked separately from prose. This does not score writing quality or factual completeness. */ +export function checkWritingMaterial(original: EvalDocuments, output: EvalDocuments): { file: string, missing: string[] }[] { + return documentLabels.map((file) => { + const code = [...original[file].matchAll(/^```[^\n]*\n[\s\S]*?^```/gm)].map(match => match[0]) + const outsideCode = original[file].replace(/^```[^\n]*\n[\s\S]*?^```/gm, '') + const identifiers = [...outsideCode.matchAll(/`([^`\n]+)`/g)].map(match => match[1]!) + const links = [...original[file].matchAll(/https?:\/\/[^\s)>'"`]+/g)].map(match => match[0]) + return { file, missing: [...new Set([...code, ...identifiers, ...links])].filter(value => !output[file].includes(value)) } + }) +} diff --git a/test/unit/writing-eval.test.ts b/test/unit/writing-eval.test.ts new file mode 100644 index 00000000..5128121e --- /dev/null +++ b/test/unit/writing-eval.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest' +import { checkWritingMaterial, parseWritingResponse } from '../../scripts/lib/writing-eval' + +const documents = { 'reading-list.md': '# Reading\n\nUse [the list](https://example.com/list).', 'README.md': '# Margin\n\nCall `filterLinks`.\n\n```ts\nfilterLinks(links, "CSS")\n```', 'pr.md': '# Change\n\nRun `pnpm test`.' } +const events = (text: string) => JSON.stringify({ type: 'text', part: { text } }) + +describe('writing evaluation responses', () => { + it('returns complete Markdown from an OpenCode response', () => { + expect(parseWritingResponse(events(JSON.stringify(documents)))).toEqual(documents) + }) + it('rejects a response with a missing document', () => { + expect(() => parseWritingResponse(events(JSON.stringify({ 'README.md': '# Only one' })))).toThrow() + }) + it('refuses a run that called a tool', () => { + const trace = `${JSON.stringify({ type: 'tool_use', part: { tool: 'bash' } })}\n${events(JSON.stringify(documents))}` + expect(() => parseWritingResponse(trace)).toThrow('tool_use') + }) + it('does not accept output after an agent error', () => { + expect(() => parseWritingResponse(`${events(JSON.stringify(documents))}\n${JSON.stringify({ type: 'error', error: 'provider failed' })}`)).toThrow('provider failed') + }) +}) + +describe('writing exact material checks', () => { + it('accepts prose edits that preserve code and links', () => { + const edited = { ...documents, 'pr.md': '# Change\n\nCheck it with `pnpm test`.' } + expect(checkWritingMaterial(documents, edited)).toEqual([ + { file: 'reading-list.md', missing: [] }, + { file: 'README.md', missing: [] }, + { file: 'pr.md', missing: [] }, + ]) + }) + it('reports changed code, missing links, and commands', () => { + const edited = { 'reading-list.md': '# Reading', 'README.md': '# Margin\n\nCall `filterLinks`.\n\n```ts\nfilterLinks([], "CSS")\n```', 'pr.md': '# Change' } + expect(checkWritingMaterial(documents, edited)).toEqual([ + { file: 'reading-list.md', missing: ['https://example.com/list'] }, + { file: 'README.md', missing: ['```ts\nfilterLinks(links, "CSS")\n```'] }, + { file: 'pr.md', missing: ['pnpm test'] }, + ]) + }) +})