diff --git a/.github/workflows/demo-rule.yml b/.github/workflows/demo-rule.yml index 8cfc9308..6bd76977 100644 --- a/.github/workflows/demo-rule.yml +++ b/.github/workflows/demo-rule.yml @@ -14,13 +14,13 @@ # this check to the protection rule. That is the same reason the house-style # scan lives inside `validate.yml` rather than beside it. # -# **It is not the only place `check.ts` runs in CI.** `test/demo-command.test.ts` -# executes it too, under `Validate`. What this job adds is a different -# assertion: it copies the bytes `taskless demo` would write — the manifest's -# own written-path list, read straight off `assets/demo-*/` — into one bare -# project, and runs the engines over them with no command and no bundle in the -# way. So a break in a shipped asset tree is attributed to that tree, rather -# than to the command that writes it or the imports that embed it. +# **It is the one place these rules' fixtures run.** The rules are the +# conformance corpus published as `@taskless/cli/reference.json`, and +# `test/reference.test.ts` only proves that payload matches the asset trees. +# This job proves the trees themselves hold: it copies the manifest's own +# written-path list, read straight off `assets/demo-*/`, into one bare project +# and runs the engines over it. A corpus rule that fails its own cases would +# hand another team a broken oracle. name: Demo rules on: @@ -84,9 +84,9 @@ jobs: pnpm --filter @taskless/cli demo:manifests /tmp/manifests.txt cat /tmp/manifests.txt # The trailing field is the manifest's own written-path list, so this - # stages exactly what `taskless demo` writes. A `cp -R` of the asset - # directory would also land `prompt.md`, which the command never - # writes, and this job would be judging a tree no user ever has. + # stages exactly the rule and its cases. A `cp -R` of the asset + # directory would also land `prompt.md`, the generation request, and + # this job would be judging a tree no generator is asked to produce. while read -r engine ruleId assetDir paths; do [ -z "$engine" ] && continue RULE="$PROJECT/.taskless/rules/$engine/$ruleId" diff --git a/eslint.config.js b/eslint.config.js index 81374cb4..010bbf67 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -17,8 +17,8 @@ export default tseslint.config( "openspec/", "**/test/fixtures/", // The demonstration rule ships as DATA, not as source. Its `check.ts` - // is bytes the CLI writes into a user's project, where it runs under - // `tsx` against a structural contract and imports nothing from this + // is a rule in the published conformance corpus, run under `tsx` + // against a structural contract and importing nothing from this // repository — so it is deliberately outside every tsconfig, and // type-aware linting has no project to resolve it against. "packages/cli/assets/demo-runtime/", diff --git a/packages/cli/scripts/print-demo-manifests.ts b/packages/cli/scripts/print-demo-manifests.ts index 84dfa3b0..f42b5ed2 100644 --- a/packages/cli/scripts/print-demo-manifests.ts +++ b/packages/cli/scripts/print-demo-manifests.ts @@ -6,10 +6,10 @@ * where a fourth sample would be added to the manifest and silently not staged. * * The trailing paths are {@link writtenPaths}, not the asset directory's - * contents, so a staging loop copies exactly what `taskless demo` writes. A - * whole-directory copy would additionally land `prompt.md`, which the command - * never writes, and the job would then be attributing results to a tree no - * user ever has. + * contents, so a staging loop copies exactly the rule and its cases. A + * whole-directory copy would additionally land `prompt.md`, which is the + * generation request rather than part of any rule tree, and the job would then + * be judging a tree a generator is never asked to produce. * * Takes an output PATH rather than printing to stdout. Capturing stdout meant * capturing whatever the runner wrapped around it: `pnpm --filter ... > file` diff --git a/packages/cli/scripts/write-reference.ts b/packages/cli/scripts/write-reference.ts index 58de1a7d..06243da2 100644 --- a/packages/cli/scripts/write-reference.ts +++ b/packages/cli/scripts/write-reference.ts @@ -5,50 +5,24 @@ * itself, and the held-out cases — separately, so another team can run their * rule against our cases and ours against theirs. * - * Reads the asset files directly rather than importing `rule.ts`, whose `?raw` - * specifiers only resolve under a vite transform. Both read `DEMO_MANIFESTS`, - * and `test/reference.test.ts` asserts they agree. + * `test/reference.test.ts` builds the payload through the same reader and + * asserts the committed file matches it. * * Run: pnpm --filter @taskless/cli reference */ -import { readFile, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { buildReference } from "../src/rules/reference"; -import { DEMO_MANIFESTS } from "../src/rules/demo/manifest"; +import { readReferenceInputs } from "../src/rules/demo/manifest"; -const packageRoot = join(import.meta.dirname, ".."); +const assetsRoot = join(import.meta.dirname, "..", "assets"); -async function readAll(assetDirectory: string, paths: readonly string[]) { - return Promise.all( - paths.map(async (path) => ({ - path, - content: await readFile( - join(packageRoot, "assets", assetDirectory, path), - "utf8" - ), - })) - ); -} - -const rules = await Promise.all( - DEMO_MANIFESTS.map(async (manifest) => ({ - engine: manifest.engine, - ruleId: manifest.ruleId, - prompt: await readFile( - join(packageRoot, "assets", manifest.assetDirectory, manifest.promptPath), - "utf8" - ), - ruleFiles: await readAll(manifest.assetDirectory, manifest.rulePaths), - testFiles: await readAll(manifest.assetDirectory, manifest.testPaths), - })) -); - -const target = join(packageRoot, "assets", "reference.json"); +const target = join(assetsRoot, "reference.json"); await writeFile( target, - `${JSON.stringify(buildReference(rules), undefined, 2)}\n`, + `${JSON.stringify(buildReference(await readReferenceInputs(assetsRoot)), undefined, 2)}\n`, "utf8" ); console.log(`Wrote ${target}`); diff --git a/packages/cli/src/commands/agent.ts b/packages/cli/src/commands/agent.ts index b9736f29..74396738 100644 --- a/packages/cli/src/commands/agent.ts +++ b/packages/cli/src/commands/agent.ts @@ -90,16 +90,6 @@ async function resolveDescription( * unrelated reasons and a bare `name === "agent"` recorded none of them. * * - `agent` is the index itself, so listing it would be circular. - * - `demo` writes a fixed example rule for someone learning what a rule is. It - * is run deliberately, in a demonstration, and it is not surface we are - * exposing yet. Its description ("Write an example rule into this project") - * is close enough to a real authoring request that an agent asked to "add a - * rule that checks X" could route to it and write the example instead. That - * is the whole reason the demonstration was originally nested under `rule`: - * the property came from not being registered at all. Making it a top-level - * verb gave it discoverability it does not want yet, and this is the cost of - * that choice, paid here. - * * - `feedback` the COMMAND is reached through a recipe, never directly. The * index lists the `feedback` and `bug-report` recipes instead (see * `FEEDBACK_TOPICS`), because they are what show the user the payload and @@ -111,7 +101,7 @@ async function resolveDescription( * Absence from this list is what puts a command in the index, so adding one is * a decision someone made rather than a step they forgot. */ -const UNLISTED_COMMANDS = new Set(["agent", "demo", "feedback"]); +const UNLISTED_COMMANDS = new Set(["agent", "feedback"]); export function createAgentCommand(subCommands: SubCommandsDef) { return defineCommand({ diff --git a/packages/cli/src/commands/demo.ts b/packages/cli/src/commands/demo.ts deleted file mode 100644 index 843ee1ba..00000000 --- a/packages/cli/src/commands/demo.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { stat } from "node:fs/promises"; -import { resolve } from "node:path"; -import process from "node:process"; - -import { defineCommand } from "citty"; - -import { ensureTasklessDirectory } from "../filesystem/directory"; -import { assessDelivery, writeDeliveredFileSet } from "../rules/deliver"; -import { type DemoRule, demoRuleFor } from "../rules/demo/rule"; -import { ruleDirectory } from "../rules/engines"; -import type { EngineName } from "../rules/layout"; -import type { CLIErrorCode } from "../types/errors"; -import { CLIError } from "../util/cli-error"; -import { getCliPrefix } from "../util/package-manager"; - -/** Report a failure the way the rule commands do, then stop. */ -function fail(message: string, code: CLIErrorCode): never { - console.error(`Error: ${message}`); - process.exitCode = 1; - throw new CLIError(message, code, { reported: true }); -} - -/** Whether a path is present, with any stat failure read as absent. */ -async function exists(path: string): Promise { - try { - await stat(path); - return true; - } catch { - return false; - } -} - -/** - * What each sample teaches, in the words the reader needs at the moment they - * have the rule on disk and are deciding what to do with it. - * - * Held next to the command rather than in the rule's own `note`, because a - * `note` is surfaced when a rule FIRES and this is guidance for someone who - * has just written one. - */ -const SUMMARIES: Record = { - sg: [ - "It flags `eval(...)` calls in TypeScript. The evidence is one expression", - "in one file, and nothing else has to be read to decide it — which is what", - "makes this an `sg` rule rather than a runtime one.", - ], - vale: [ - "It flags `utilize` in markdown and suggests `use`. Vale reads prose, so", - "the rule is scoped to `**/*.md` and never looks at source.", - ], - runtime: [ - "It flags any `process.env` read whose key is not declared in the", - "repository-root `.env`. Deciding that needs both files at once, which is", - "what makes it a runtime rule rather than a pattern.", - ], -}; - -/** Write one shipped demo rule into the project. */ -async function writeDemoRule(cwd: string, rule: DemoRule): Promise { - await ensureTasklessDirectory(cwd); - - const directory = ruleDirectory(cwd, rule.engine, rule.ruleId); - - // Refused rather than merged or overwritten. This is the one destructive - // thing the command could do, and a demonstration is never worth doing it to - // a rule someone has edited. - if (await exists(directory)) { - fail( - `${rule.engine}/${rule.ruleId} already exists in this project. ` + - `Nothing was written. Remove it with ` + - `\`${getCliPrefix()} rule delete ${rule.ruleId}\` and run this again.`, - "RULE_EXISTS" - ); - } - - // Through the delivery path, not a second writer: the path-traversal and - // completeness guards a served rule gets are the ones these rules want too. - const assessment = assessDelivery(cwd, rule.engine, rule.ruleId, rule.files); - if (!assessment.ok) { - // Unreachable from shipped bytes; reported rather than asserted so a future - // edit to a manifest fails loudly instead of writing a half-formed rule. - fail( - `The bundled ${rule.engine} demo rule was refused: ${assessment.reason}`, - "INTERNAL_ERROR" - ); - } - await writeDeliveredFileSet(cwd, rule.engine, rule.ruleId, assessment); - - const prefix = getCliPrefix(); - console.log(`Wrote ${rule.engine}/${rule.ruleId}.`); - console.log(""); - for (const line of SUMMARIES[rule.engine]) console.log(line); - console.log(""); - - if (rule.engine === "runtime") { - console.log("Run its fixtures:"); - console.log(` ${prefix} test --dangerously-run-scripts`); - console.log(""); - console.log( - `\`${prefix} check\` will NOT run it. A runtime rule executes only when an` - ); - console.log( - "authenticated reconcile returns its signature, and this rule was written" - ); - console.log( - "locally rather than issued, so no run set contains it. That is the gate" - ); - console.log("working, not a failure."); - } else { - console.log("Run its fixtures:"); - console.log(` ${prefix} test`); - console.log(""); - console.log( - `\`${prefix} check\` runs it too: ${rule.engine} rules are inert data, so` - ); - console.log("there is nothing to gate."); - } - - console.log(""); - console.log("Remove it:"); - console.log(` ${prefix} rule delete ${rule.ruleId}`); -} - -/** One subcommand per engine, so `demo --help` lists what can be written. */ -function engineDemoCommand(engine: EngineName, description: string) { - return defineCommand({ - meta: { name: engine, description }, - args: { - dir: { type: "string", alias: "d", description: "Working directory" }, - }, - async run({ args }) { - const cwd = resolve(args.dir ?? process.cwd()); - const rule = demoRuleFor(engine); - if (rule === undefined) { - fail(`No demo rule ships for engine "${engine}"`, "INTERNAL_ERROR"); - } - await writeDemoRule(cwd, rule); - }, - }); -} - -export const demoCommand = defineCommand({ - meta: { - name: "demo", - description: "Write an example rule into this project", - }, - subCommands: { - sg: engineDemoCommand( - "sg", - "An ast-grep rule: one expression, one file, no execution" - ), - vale: engineDemoCommand("vale", "A Vale prose rule over markdown"), - runtime: engineDemoCommand( - "runtime", - "A runtime rule whose evidence spans two files" - ), - }, -}); diff --git a/packages/cli/src/commands/names.ts b/packages/cli/src/commands/names.ts index 28ab4a1c..142fbe94 100644 --- a/packages/cli/src/commands/names.ts +++ b/packages/cli/src/commands/names.ts @@ -13,7 +13,6 @@ export const SUBCOMMAND_NAMES = [ "agent", "auth", "check", - "demo", "detect", "feedback", "info", diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index ea91fc1e..5d893f74 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -6,7 +6,6 @@ import { z } from "zod"; import { createAgentCommand } from "./commands/agent"; import { authCommand } from "./commands/auth"; import { checkCommand } from "./commands/check"; -import { demoCommand } from "./commands/demo"; import { detectCommand } from "./commands/detect"; import { feedbackCommand } from "./commands/feedback"; import { initCommand, updateCommand } from "./commands/init"; @@ -53,7 +52,6 @@ const subCommands = { info: infoCommand, detect: detectCommand, check: checkCommand, - demo: demoCommand, feedback: feedbackCommand, auth: authCommand, onboard: onboardCommand, diff --git a/packages/cli/src/rules/demo/manifest.ts b/packages/cli/src/rules/demo/manifest.ts index 50787feb..7366eab8 100644 --- a/packages/cli/src/rules/demo/manifest.ts +++ b/packages/cli/src/rules/demo/manifest.ts @@ -1,16 +1,14 @@ /** - * The demonstration rules' identities and file lists, with no imports at all. + * The demonstration rules behind the conformance corpus: their identities, + * their file lists, and the one reader that loads them off disk. * - * Separate from `rule.ts` because that module embeds the file CONTENTS through - * vite's `?raw`, which only resolves under a vite transform. A plain Node - * script — the reference-payload generator — cannot import it, and neither can - * anything else outside the bundle. - * - * The paths therefore live here, stated once. `rule.ts` embeds exactly these - * lists, the generator reads exactly these lists off disk, and - * `test/reference.test.ts` asserts the two agree — so the rules the CLI - * writes and the payload handed to other teams cannot come to describe - * different things. + * Nothing in the bundle imports this. The rules used to be written into a + * project by `taskless demo`; that command is gone, and what remains is the + * corpus published as `@taskless/cli/reference.json`. The generator script and + * `test/reference.test.ts` both build that payload through + * {@link readReferenceInputs}, so "the committed file is current" is a + * comparison against a fresh regeneration rather than against a second copy of + * the lists. * * ## Why there are three * @@ -21,7 +19,11 @@ * cannot: why there are three tiers at all. */ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + import type { EngineName } from "../layout"; +import type { ReferenceInput } from "../reference"; /** One shipped demonstration rule. */ export interface DemoManifest { @@ -31,12 +33,12 @@ export interface DemoManifest { * * Fixed, with no batch suffix. A generated rule carries one so a * regeneration does not collide with what is already there; these are - * written from constant bytes, so writing one twice would be the same rule. + * constant bytes, so there is nothing to collide with. */ ruleId: string; /** * The generation request this rule is the answer to, as a caller would phrase - * it. Lives beside the rule but is NEVER written into a project. + * it. Lives beside the rule but is never part of the staged rule tree. * * Published so another team can generate from the same input and compare. * Without it a reference payload shows what we produced and not what we were @@ -104,7 +106,32 @@ export const DEMO_MANIFESTS: readonly DemoManifest[] = [ }, ]; -/** Every file written into a project: the rule and its cases, never the prompt. */ +/** Every file of the staged rule tree: the rule and its cases, never the prompt. */ export function writtenPaths(manifest: DemoManifest): readonly string[] { return [...manifest.rulePaths, ...manifest.testPaths]; } + +/** Every manifest's prompt, rule and cases, read from `assetsRoot`. */ +export async function readReferenceInputs( + assetsRoot: string +): Promise { + const read = (assetDirectory: string, path: string) => + readFile(join(assetsRoot, assetDirectory, path), "utf8"); + const readAll = (assetDirectory: string, paths: readonly string[]) => + Promise.all( + paths.map(async (path) => ({ + path, + content: await read(assetDirectory, path), + })) + ); + + return Promise.all( + DEMO_MANIFESTS.map(async (manifest) => ({ + engine: manifest.engine, + ruleId: manifest.ruleId, + prompt: await read(manifest.assetDirectory, manifest.promptPath), + ruleFiles: await readAll(manifest.assetDirectory, manifest.rulePaths), + testFiles: await readAll(manifest.assetDirectory, manifest.testPaths), + })) + ); +} diff --git a/packages/cli/src/rules/demo/rule.ts b/packages/cli/src/rules/demo/rule.ts deleted file mode 100644 index 2ca12fcd..00000000 --- a/packages/cli/src/rules/demo/rule.ts +++ /dev/null @@ -1,141 +0,0 @@ -/** - * The demonstration rules, embedded from `assets/demo-*`. - * - * ## Why these ship rather than being generated - * - * The demonstration used to be fetched from a service endpoint, so that it - * exercised the same generation path a real request does. That reasoning was - * sound about generation and wrong about what the demonstration is for: it made - * a demonstration of the TIER depend on a demonstration of GENERATION, and - * every failure mode of the second became a failure mode of the first. - * - * It was not hypothetical. The served rule arrived classified as `sg` — a tier - * whose evidence fits in one file — although the fixture behind it exists - * precisely because its subject spans two. Its two-file passing example reached - * the client with the second file deleted, at which point it matched the - * single-file pattern it had been flattened into, and the rule failed - * `taskless verify` on arrival. - * - * A rule that ships cannot arrive mis-tiered, flattened, or with an id its body - * does not match, because CI runs `taskless verify` over these exact bytes. - * - * ## Why the imports are written out - * - * `?raw` is a vite transform, so the specifiers must be literals it can see at - * build time — `import.meta.glob` would skip the dot-paths, and a computed - * specifier would not be transformed at all. `manifest.ts` holds the lists; - * this module is where each entry acquires its bytes. - */ - -import sgPrompt from "../../../assets/demo-sg/prompt.md?raw"; -import sgRule from "../../../assets/demo-sg/no-eval-call.yml?raw"; -import sgTest from "../../../assets/demo-sg/.tests/no-eval-call-test.yml?raw"; - -import valePrompt from "../../../assets/demo-vale/prompt.md?raw"; -import valeRule from "../../../assets/demo-vale/prefer-use-over-utilize.yml?raw"; -import valeConfig from "../../../assets/demo-vale/.vale.ini?raw"; -import valePass from "../../../assets/demo-vale/.tests/pass/README.md?raw"; -import valeFail from "../../../assets/demo-vale/.tests/fail/README.md?raw"; - -import runtimePrompt from "../../../assets/demo-runtime/prompt.md?raw"; -import captureEnvironmentRead from "../../../assets/demo-runtime/captures/env-read.yml?raw"; -import checkSource from "../../../assets/demo-runtime/check.ts?raw"; -import failEnvironment from "../../../assets/demo-runtime/.tests/fail/undeclared/.env?raw"; -import failSource from "../../../assets/demo-runtime/.tests/fail/undeclared/src/config.ts?raw"; -import passEnvironment from "../../../assets/demo-runtime/.tests/pass/declared/.env?raw"; -import passSource from "../../../assets/demo-runtime/.tests/pass/declared/src/config.ts?raw"; - -import type { EngineName } from "../layout"; -import type { DeliveredFile } from "../deliver"; -import { DEMO_MANIFESTS, writtenPaths } from "./manifest"; - -/** One shipped demonstration rule, with the bytes of every file it contains. */ -export interface DemoRule { - engine: EngineName; - ruleId: string; - /** The generation request, published for conformance and never written. */ - prompt: string; - /** The rule itself: what a generator has to produce. */ - ruleFiles: readonly DeliveredFile[]; - /** The held-out cases both sides' rules must satisfy. */ - testFiles: readonly DeliveredFile[]; - /** - * Everything written into a project, rule and cases together. - * - * Deliberately the same `DeliveredFile[]` a served rule produces, so a demo - * is written by `assessDelivery` + `writeDeliveredFileSet` rather than by a - * second writer. If one ever needs its own writer, the shapes have diverged - * and that is the finding. - */ - files: readonly DeliveredFile[]; -} - -/** Contents by asset path, keyed exactly as `manifest.ts` lists them. */ -const PROMPTS: Record = { - "demo-sg": sgPrompt, - "demo-vale": valePrompt, - "demo-runtime": runtimePrompt, -}; - -const CONTENTS: Record> = { - "demo-sg": { - "no-eval-call.yml": sgRule, - ".tests/no-eval-call-test.yml": sgTest, - }, - "demo-vale": { - "prefer-use-over-utilize.yml": valeRule, - ".vale.ini": valeConfig, - ".tests/pass/README.md": valePass, - ".tests/fail/README.md": valeFail, - }, - "demo-runtime": { - "check.ts": checkSource, - "captures/env-read.yml": captureEnvironmentRead, - ".tests/pass/declared/src/config.ts": passSource, - ".tests/pass/declared/.env": passEnvironment, - ".tests/fail/undeclared/src/config.ts": failSource, - ".tests/fail/undeclared/.env": failEnvironment, - }, -}; - -/** - * Every shipped demonstration rule. - * - * Built by walking the manifests rather than by restating them, so a path added - * to a manifest without a matching embed throws HERE, at module load, rather - * than producing a rule that writes one file fewer than it should. - */ -function embed(assetDirectory: string, ruleId: string) { - return (path: string): DeliveredFile => { - const content = CONTENTS[assetDirectory]?.[path]; - if (content === undefined) { - throw new Error( - `demo rule ${ruleId} lists ${path} in its manifest but nothing embeds it` - ); - } - return { path, content }; - }; -} - -export const DEMO_RULES: readonly DemoRule[] = DEMO_MANIFESTS.map( - (manifest) => { - const bytes = embed(manifest.assetDirectory, manifest.ruleId); - const prompt = PROMPTS[manifest.assetDirectory]; - if (prompt === undefined) { - throw new Error(`demo rule ${manifest.ruleId} embeds no prompt`); - } - return { - engine: manifest.engine, - ruleId: manifest.ruleId, - prompt, - ruleFiles: manifest.rulePaths.map((path) => bytes(path)), - testFiles: manifest.testPaths.map((path) => bytes(path)), - files: writtenPaths(manifest).map((path) => bytes(path)), - }; - } -); - -/** The shipped demonstration rule for one engine, if there is one. */ -export function demoRuleFor(engine: string): DemoRule | undefined { - return DEMO_RULES.find((rule) => rule.engine === engine); -} diff --git a/packages/cli/test/agent-extensions.test.ts b/packages/cli/test/agent-extensions.test.ts index 6a619745..6b3d5f05 100644 --- a/packages/cli/test/agent-extensions.test.ts +++ b/packages/cli/test/agent-extensions.test.ts @@ -66,29 +66,6 @@ describe("taskless agent (no args)", () => { } }); - it("does not advertise an unlisted command", async () => { - const result = await runCli(["agent", "-d", cwd]); - - // `demo` writes a fixed example rule for someone learning what a rule is. - // It is run deliberately, in a demonstration, and is not surface we expose - // yet. The risk is specific rather than tidiness: its description is close - // enough to a real authoring request that an agent asked to "add a rule - // that checks X" could route to it and write the example instead. - // - // Asserted against the WHOLE index rather than the Topics block, because - // the index has two independently-sourced listings and appearing in either - // would advertise it. - expect(result.stdout).not.toContain("demo"); - }); - - it("still dispatches a command it does not advertise", async () => { - // Unlisted is not unavailable. Without this, deleting the command outright - // would satisfy the assertion above. - const result = await runCli(["demo", "--help"]); - expect(result.exitCode).toBe(0); - expect(result.stdout).toContain("runtime"); - }); - it("mentions the --anonymous flag", async () => { const result = await runCli(["agent", "-d", cwd]); expect(result.stdout).toContain("--anonymous"); diff --git a/packages/cli/test/demo-command.test.ts b/packages/cli/test/demo-command.test.ts deleted file mode 100644 index b1e0b5a3..00000000 --- a/packages/cli/test/demo-command.test.ts +++ /dev/null @@ -1,300 +0,0 @@ -import { execFile } from "node:child_process"; -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { promisify } from "node:util"; - -import { afterEach, beforeEach, describe, expect, it } from "vitest"; - -import { DEMO_RULES, demoRuleFor } from "../src/rules/demo/rule"; -import { builtCli } from "./support/built-cli"; - -const execFileAsync = promisify(execFile); -const binPath = builtCli(); - -/** - * The demonstration rules, end to end over the built CLI. - * - * Over the BUILT CLI on purpose. These reach a user as bytes embedded in the - * bundle, and several of their files live under `.tests/` or are named `.env` - * or `.vale.ini` — dot-paths that glob helpers skip by default. A unit test - * importing the module would confirm the imports resolve and say nothing about - * whether the bundle carries them. - */ - -let project: string; - -async function run( - args: string[] -): Promise<{ stdout: string; stderr: string; exitCode: number }> { - try { - const { stdout, stderr } = await execFileAsync("node", [binPath, ...args]); - return { stdout, stderr, exitCode: 0 }; - } catch (error) { - const failure = error as { - stdout?: string; - stderr?: string; - code?: number; - }; - return { - stdout: failure.stdout ?? "", - stderr: failure.stderr ?? "", - exitCode: failure.code ?? 1, - }; - } -} - -interface Verdict { - ruleId: string; - ok: boolean; - ran: boolean; - errors: string[]; -} - -/** Every rule's verdict from one `taskless verify --json` run. */ -async function verdicts(extra: string[] = []): Promise { - const { stdout } = await run(["verify", "--json", ...extra, "-d", project]); - return (JSON.parse(stdout) as { rules: Verdict[] }).rules; -} - -async function verdictFor( - ruleId: string, - extra: string[] = [] -): Promise { - const all = await verdicts(extra); - const found = all.find((rule) => rule.ruleId === ruleId); - if (found === undefined) throw new Error(`no report for ${ruleId}`); - return found; -} - -beforeEach(async () => { - project = await mkdtemp(join(tmpdir(), "tskl-demo-")); - await run(["init", "-d", project]); -}); - -afterEach(async () => { - await rm(project, { recursive: true, force: true }); -}); - -describe.each(DEMO_RULES.map((rule) => [rule.engine, rule.ruleId] as const))( - "taskless demo %s", - (engine, ruleId) => { - it("writes a rule that validates", async () => { - const written = await run(["demo", engine, "-d", project]); - expect(written.exitCode).toBe(0); - - const verified = await run(["validate", "-d", project]); - expect(verified.exitCode).toBe(0); - expect(verified.stdout).toContain(`${engine}/${ruleId}`); - }); - - it("writes every file verbatim, dot-paths included", async () => { - await run(["demo", engine, "-d", project]); - const rule = demoRuleFor(engine); - expect(rule).toBeDefined(); - - expect( - rule?.files.some((file) => file.path.endsWith("prompt.md")), - "the generation prompt must never be written into a project" - ).toBe(false); - - for (const file of rule?.files ?? []) { - const onDisk = await readFile( - join(project, ".taskless/rules", engine, ruleId, file.path), - "utf8" - ); - expect(onDisk, `${file.path} was not written verbatim`).toBe( - file.content - ); - } - }); - - it("refuses to overwrite a rule that is already there", async () => { - await run(["demo", engine, "-d", project]); - - const rule = demoRuleFor(engine); - const first = rule?.files[0]?.path ?? ""; - const target = join(project, ".taskless/rules", engine, ruleId, first); - await writeFile(target, "edited by hand\n"); - - const second = await run(["demo", engine, "-d", project]); - expect(second.exitCode).toBe(1); - expect(second.stderr).toContain("already exists"); - - // The refusal has to be a refusal, not a message before a write. - expect(await readFile(target, "utf8")).toBe("edited by hand\n"); - }); - - it("can be removed, leaving nothing a later check reports", async () => { - await run(["demo", engine, "-d", project]); - const deleted = await run(["rule", "delete", ruleId, "-d", project]); - expect(deleted.exitCode).toBe(0); - - const { stdout, stderr } = await run(["check", "-d", project]); - expect(stdout + stderr).not.toContain(ruleId); - }); - } -); - -describe("the three samples together", () => { - it("all pass their own fixtures in one project", async () => { - for (const rule of DEMO_RULES) { - await run(["demo", rule.engine, "-d", project]); - } - - const all = await verdicts(["--dangerously-run-scripts"]); - expect(all).toHaveLength(DEMO_RULES.length); - for (const verdict of all) { - // `ran` matters as much as `ok`: a rule reporting a pass for fixtures - // that never executed is the defect these runners exist to stop - // reporting. - expect( - verdict.ok, - `${verdict.ruleId}: ${verdict.errors.join("; ")}` - ).toBe(true); - expect(verdict.ran, `${verdict.ruleId} never ran`).toBe(true); - } - }); - - it("covers one engine each, which is the point of shipping three", () => { - expect([...new Set(DEMO_RULES.map((rule) => rule.engine))]).toHaveLength( - DEMO_RULES.length - ); - }); -}); - -describe("the runtime sample does not weaken the execution gate", () => { - beforeEach(async () => { - await run(["demo", "runtime", "-d", project]); - }); - - it("is skipped by an unauthenticated check, with the unauthenticated reason", async () => { - const { stdout, stderr, exitCode } = await run(["check", "-d", project]); - expect(exitCode).toBe(0); - // The skip is a NOTICE on stderr, not a finding on stdout: `check` found no - // issues, and reporting a rule it declined to run as an issue would be the - // opposite of what the gate means. - expect(stdout).toContain("No issues found."); - expect(stderr).toContain("env-keys-declared"); - expect(stderr).toContain("not authenticated, so it was not verified"); - }); - - it("does not execute its fixtures without the documented flag", async () => { - const verdict = await verdictFor("env-keys-declared"); - expect(verdict.ran).toBe(false); - }); -}); - -/** A path inside one written rule's `.tests/`. */ -function fixture( - root: string, - engine: string, - ruleId: string, - ...parts: string[] -): string { - return join(root, ".taskless/rules", engine, ruleId, ".tests", ...parts); -} - -describe("the samples' fixtures are load-bearing", () => { - // Each mutation breaks one half of one claim, and must break exactly that - // rule. Without these a fixture pair could rot into something that passes - // because it asserts nothing. - - it("sg fails when its valid case starts matching", async () => { - await run(["demo", "sg", "-d", project]); - await writeFile( - fixture(project, "sg", "no-eval-call", "no-eval-call-test.yml"), - "id: no-eval-call\nvalid:\n - const r = eval(p);\ninvalid:\n - const r = eval(p);\n" - ); - const verdict = await verdictFor("no-eval-call"); - expect(verdict.ok).toBe(false); - }); - - it("vale fails when its pass fixture starts firing", async () => { - await run(["demo", "vale", "-d", project]); - await writeFile( - fixture(project, "vale", "prefer-use-over-utilize", "pass", "README.md"), - "# Setup\n\nUtilize the installer.\n" - ); - const verdict = await verdictFor("prefer-use-over-utilize"); - expect(verdict.ok).toBe(false); - expect(verdict.errors.join("\n")).toContain("pass fixture wrongly fired"); - }); - - it("vale fails when its fail fixture stops firing", async () => { - await run(["demo", "vale", "-d", project]); - await writeFile( - fixture(project, "vale", "prefer-use-over-utilize", "fail", "README.md"), - "# Setup\n\nUse the installer.\n" - ); - const verdict = await verdictFor("prefer-use-over-utilize"); - expect(verdict.ok).toBe(false); - expect(verdict.errors.join("\n")).toContain("fail fixture did not fire"); - }); - - it("runtime fails when its fail case stops firing", async () => { - await run(["demo", "runtime", "-d", project]); - await writeFile( - fixture( - project, - "runtime", - "env-keys-declared", - "fail", - "undeclared", - ".env" - ), - "API_URL=https://api.example.com\nAPI_KEY=declared-now\n" - ); - const verdict = await verdictFor("env-keys-declared", [ - "--dangerously-run-scripts", - ]); - expect(verdict.ok).toBe(false); - expect(verdict.errors.join("\n")).toContain("fail fixture did not fire"); - }); - - it("runtime fails when its pass case starts firing", async () => { - await run(["demo", "runtime", "-d", project]); - await writeFile( - fixture( - project, - "runtime", - "env-keys-declared", - "pass", - "declared", - ".env" - ), - "# nothing declared\n" - ); - const verdict = await verdictFor("env-keys-declared", [ - "--dangerously-run-scripts", - ]); - expect(verdict.ok).toBe(false); - expect(verdict.errors.join("\n")).toContain("pass fixture wrongly fired"); - }); - - it("runtime fails when its pass case no longer matches the captures", async () => { - // The runtime-specific one, and the reason an `sg` reading of that tier is - // wrong: a clean case must still CONTAIN the construct, or the captures do - // not fire and the check never gets to judge it. - await run(["demo", "runtime", "-d", project]); - await writeFile( - fixture( - project, - "runtime", - "env-keys-declared", - "pass", - "declared", - "src", - "config.ts" - ), - "export const apiUrl = config.apiUrl;\n" - ); - const verdict = await verdictFor("env-keys-declared", [ - "--dangerously-run-scripts", - ]); - expect(verdict.ok).toBe(false); - expect(verdict.errors.join("\n")).toContain( - "matched no captures, so check.ts never ran" - ); - }); -}); diff --git a/packages/cli/test/dogfood-scaffold-current.test.ts b/packages/cli/test/dogfood-scaffold-current.test.ts index 6016038d..d20952fb 100644 --- a/packages/cli/test/dogfood-scaffold-current.test.ts +++ b/packages/cli/test/dogfood-scaffold-current.test.ts @@ -7,8 +7,8 @@ import { LATEST_SCHEMA_VERSION } from "../src/filesystem/migrate"; * This repository's own `.taskless/` is on the scaffold version this CLI * ships. * - * Migrations run on `init`, `demo`, `onboard`, and rule delivery, and on - * nothing the development loop invokes: `pnpm lint` builds and runs `check`, + * Migrations run on `init`, `onboard`, and rule delivery, and on nothing the + * development loop invokes: `pnpm lint` builds and runs `check`, * which refuses a stale scaffold rather than migrating it. So a pull request * that adds a migration can pass every other check while leaving the dogfood * tree behind, and the gap only shows when someone's `check` hits the wall. diff --git a/packages/cli/test/reference.test.ts b/packages/cli/test/reference.test.ts index 25c81205..42f6efc0 100644 --- a/packages/cli/test/reference.test.ts +++ b/packages/cli/test/reference.test.ts @@ -10,26 +10,21 @@ import { type ReferenceInput, } from "../src/rules/reference"; import { RULE_CONSTRAINTS } from "../src/rules/constraints"; -import { DEMO_MANIFESTS, writtenPaths } from "../src/rules/demo/manifest"; +import { + DEMO_MANIFESTS, + readReferenceInputs, + writtenPaths, +} from "../src/rules/demo/manifest"; import { ENGINE_LAYOUTS, ENGINES, type EngineName } from "../src/rules/layout"; import { ruleDirectory } from "../src/rules/engines"; -import { DEMO_RULES } from "../src/rules/demo/rule"; /** - * The reference payload other teams consume, and the rules we ship, describe - * the same thing. - * - * Two lists exist because `?raw` only resolves under a vite transform, so the - * generator script reads the assets off disk while the bundle embeds them. - * Nothing stops those from drifting except this file. + * The reference payload other teams consume, and the asset trees it is built + * from, describe the same thing. */ -const referencePath = join( - import.meta.dirname, - "..", - "assets", - "reference.json" -); +const assetsRoot = join(import.meta.dirname, "..", "assets"); +const referencePath = join(assetsRoot, "reference.json"); async function readReference(): Promise { return JSON.parse(await readFile(referencePath, "utf8")) as Reference; @@ -100,19 +95,9 @@ function shapePaths(value: unknown): string[] { describe("the demo reference payload", () => { it("is current — regenerate with `pnpm --filter @taskless/cli reference`", async () => { - expect(await readReference()).toEqual(buildReference(DEMO_RULES)); - }); - - it("embeds exactly the files each manifest lists", () => { - for (const manifest of DEMO_MANIFESTS) { - const rule = DEMO_RULES.find((entry) => entry.engine === manifest.engine); - expect(rule?.ruleFiles.map((file) => file.path)).toEqual([ - ...manifest.rulePaths, - ]); - expect(rule?.testFiles.map((file) => file.path)).toEqual([ - ...manifest.testPaths, - ]); - } + expect(await readReference()).toEqual( + buildReference(await readReferenceInputs(assetsRoot)) + ); }); it("names every dot-path explicitly, because a glob would drop them", () => {