diff --git a/.changeset/streamlined-onboarding-report.md b/.changeset/streamlined-onboarding-report.md index d71eec5a..c959c40c 100644 --- a/.changeset/streamlined-onboarding-report.md +++ b/.changeset/streamlined-onboarding-report.md @@ -4,6 +4,6 @@ `taskless detect --json` now reports ast-grep and Vale configurations as linters (Taskless's own configuration under `.taskless/` excluded), the command-line tools on the host's `PATH` (`gh`, `git`, `jq`, and now `glab` for GitLab repositories) under `hostTools`, and an `agent` section of commands an agent may choose to run to check whether a source is usable, such as `gh auth status`. Detection still runs nothing and stays offline. `taskless info` lists `glab` among its tools too. -New `taskless report` publishes an unlisted Taskless report to tskl.es: a page showing the rules in this repository, what they were drawn from, and what they are projected to catch. With no flags it prints a draft of what the CLI knows and the facts an agent still has to supply; `--from ` checks there are at least three working Taskless rules, withholds any fixture that looks like it holds a secret or contact detail, validates against the tskl.es schema, publishes, and records the report in `.taskless/reports/` (gitignored, since each record holds the URL that deletes it); `--delete ` takes a report down. The new `report` agent topic walks an agent through it. Migration 11 creates `.taskless/reports/`. +New `taskless report` publishes an unlisted Taskless report to tskl.es: a page showing the rules in this repository, what they find in the code today, and how often review raised each one's pattern. The CLI counts what the rules find itself, by running the same scan as `check --anonymous`, and an agent cannot supply or change those numbers. When the CLI could not run every rule (a rule written into another linter, a runtime rule, or a scan that failed), it publishes no counts rather than a partial total. With no flags it prints a draft of what the CLI knows, including those counts, and the facts an agent still has to supply; `--from ` checks there are at least three working Taskless rules, withholds any fixture that looks like it holds a secret or contact detail, validates against the tskl.es schema, publishes, and records the report in `.taskless/reports/` (gitignored, since each record holds the URL that deletes it); `--delete ` takes a report down. The new `report` agent topic walks an agent through it. Migration 11 creates `.taskless/reports/`. Onboarding now asks one question, "Do you want me to set up Taskless for you?", and does the rest: it mines every source it can reach, writes every rule the repository can enforce locally (always logged out), marks onboarding complete, publishes a report, sends Taskless onboarding feedback, and ends by strongly encouraging running the rules in CI. It asks anything else only when it cannot find enough rules for a report. GitLab merge requests are mined through `glab` the way GitHub pull requests are through `gh`. diff --git a/openspec/changes/archive/2026-10-14-report-measured-findings/.openspec.yaml b/openspec/changes/archive/2026-10-14-report-measured-findings/.openspec.yaml new file mode 100644 index 00000000..49ea85b3 --- /dev/null +++ b/openspec/changes/archive/2026-10-14-report-measured-findings/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-10-14 diff --git a/openspec/changes/archive/2026-10-14-report-measured-findings/design.md b/openspec/changes/archive/2026-10-14-report-measured-findings/design.md new file mode 100644 index 00000000..89ec4a37 --- /dev/null +++ b/openspec/changes/archive/2026-10-14-report-measured-findings/design.md @@ -0,0 +1,69 @@ +## Context + +`taskless check` runs its whole scan inside `checkCommand.run`: path classification, the schema refusal, `--rule` selection, the run directory, `planCheck`, engine config assembly, `runEngines`, and output. None of it is callable from another command. `taskless report` already runs every rule's tests through `verifyOneRule` while building its draft, but has never run the scan. + +The live ReportV1 schema (fetched 2026-10-14) is the authority for the payload shape: `findings`, per-rule `state`, `violations`, `files`, and `surfaced`; no `fixed`, `fixedBasis`, `origin`, or `projection.perMonth`. Its rule: `findings` is present exactly when every written rule carries `violations` and `files`. Its service-side consistency checks require `findings.violations` to equal the rules' sum and `findings.files` to fall between the largest rule's `files` and their sum. + +## Goals / Non-Goals + +**Goals:** + +- `report` and `check` run the same scan code, so a report's counts are the counts `check --anonymous` would print. +- Counts are deterministic: the CLI either has a defensible number for every written rule or publishes none. + +**Non-Goals:** + +- Running a legacy rule's own tool. The CLI cannot run eslint or ruff, and taking the agent's word for those counts is what this change exists to stop. +- Running runtime rules for a report. Reports are published logged out. +- Recomputing tskl.es's monthly rates in the terminal summary. + +## Decisions + +### Extract the scan, keep `check`'s surface + +Move the part of `check` between "rules exist" and "results in hand" (open the run directory, `planCheck`, `assembleEngineConfigs`, `runEngines`, close the run) into a function under `packages/cli/src/rules/` that returns the plan and the dispatch. `check` keeps its argument handling, notices, output, exit code, and telemetry, and calls the function. `report` calls it with `anonymous: true`, no paths, and no selection. + +`check` prints the plan's notices before dispatch; after the extraction they are printed when the function returns. Only stderr ordering moves, and only in text mode; `--json` output is unchanged. + +_Alternative:_ spawn the built CLI's `check --json` from `report`. Rejected: it needs the built binary at a known path, doubles startup, and a test of `report` would depend on a build. + +_Alternative:_ have `report` assemble its own sg and Vale invocation. Rejected: two scans that can drift apart are exactly how a report's number stops matching `check`'s. + +### Anonymous is what keeps the draft offline + +`planCheck` with `anonymous: true` skips reconcile entirely, so the scan makes no network call and every runtime rule is skipped. This is what lets the draft keep its "no network" requirement. + +### All-or-nothing, decided in one place + +A single function takes the written rules and the scan outcome and returns either `{ perRule, findings }` or nothing. It returns nothing when any of these holds: + +- a written rule is `legacy`, or is a runtime rule under `.taskless/rules/runtime/`; +- the plan or the dispatch reports a failure; +- a result's `ruleId` names no rule under `.taskless/rules/sg/` or `.taskless/rules/vale/`. + +The last guard matters because ast-grep reports a result under the `id:` in the rule body, and `sg-id-matches-directory` is a constraint `check` reports rather than a guarantee. Without the guard, a mismatched rule would publish a confident 0 while its real hits sat under a name nothing in the report has. Vale results already carry the bare directory id. + +The draft computes this from the local rules alone. `--from` re-applies it after merging the facts, since a `legacy` entry in the facts is what removes the counts. + +### File identity is the path relative to the repository root + +Each result's `file` is resolved against `cwd` and made relative again before it is counted, so `./src/a.ts` and `src/a.ts` are one file. The service's lower bound (`findings.files` is at least the largest rule's `files`) then holds by construction. + +### State comes from where a rule came from + +`state` is set by the assembler, never by the facts: rules under `.taskless/rules/` and facts-file `legacy` entries are `written`, facts-file `cloud` entries are `candidate`. A local runtime rule is `written`, and it is the rule that triggers the runtime guard above. + +### The summary reports raw totals, not rates + +The summary prints `findings` when present, then Σ `surfaced.reviewComments` across Σ `surfaced.prs` over the window. It does not reproduce the service's ×30 ÷ days formula: two copies of a formula in two repositories is a number that drifts silently. + +### The counting method counts every author + +A comment asking for a rule's pattern costs something whoever wrote it: a bot review spends tokens and money, an author's own fix-up note spends their time and an agent's cycles, and a reviewer's spends review time. All three are what the rule saves, so bots and the pull request's author count the same as a reviewer. Excluding them was considered and rejected: it would undercount exactly the agent-heavy repositories a report is most often published for. The agent classifies each comment by reading it, because the bug report's run showed keyword matching is what an agent improvises when given no method. + +## Risks / Trade-offs + +- [The draft gets slower: it now scans the whole project as well as running tests.] → The draft runs once per report, and the scan is the same one CI already pays for. +- [A repository with one legacy rule never shows a findings headline.] → Accepted. tskl.es has a headline for that case, and a partly measured total would be a number the reader cannot interpret. +- [Moving plan notices after dispatch changes `check`'s stderr order.] → Text mode only, and nothing parses it. `--json` is unaffected, and its tests guard it. +- [A rule whose tests fail still runs in the scan and is counted.] → Correct: its matches are real matches. The gate already decides whether a failing rule counts toward the three. diff --git a/openspec/changes/archive/2026-10-14-report-measured-findings/proposal.md b/openspec/changes/archive/2026-10-14-report-measured-findings/proposal.md new file mode 100644 index 00000000..1491da6e --- /dev/null +++ b/openspec/changes/archive/2026-10-14-report-measured-findings/proposal.md @@ -0,0 +1,36 @@ +## Why + +A published report never says what its rules find in the code today, even though `taskless check` measures exactly that. Every number on the page came from the facts file, so the one figure that does not depend on trusting the agent was missing, and `fixedBasis: measured` could be claimed with no method behind it. tskl.es has since reshaped ReportV1 around measured findings (live at `https://tskl.es/api/v1/schema` on 2026-10-14), and the nightly's payload no longer validates against it: every publish is refused with a 400. + +## What Changes + +- **The CLI measures current violations itself.** `taskless report` runs the project's static rules the way `check --anonymous` does and records, per written rule, `violations` and `files`, plus a top-level `findings: { violations, files }` where `files` counts distinct files across all rules. The draft carries them, and `--from` re-measures at publish. +- **Counts are all-or-nothing.** They are published only when the CLI ran every written rule and could attribute every result to one. A legacy rule, a local runtime rule, an engine failure, or a result under an unknown rule id omits `findings` and every per-rule count, and tskl.es falls back to a headline built from what remains. The agent can never supply them. +- **Every rule carries `state`.** Rules under `.taskless/rules/` and legacy rules are `written`; facts-file `cloud` entries are `candidate` and carry no counts. +- **BREAKING (nightly only):** `fixed` and `fixedBasis` are removed from the facts file and the payload. `origin` is renamed `surfaced: { prs, reviewComments }`. +- **BREAKING (nightly only):** `projection.perMonth` is no longer sent. tskl.es derives both monthly rates from each rule's `surfaced` and the window, and refuses a payload that carries them. `projection.inputs` is unchanged. +- The vendored ReportV1 schema is refreshed. +- The publish summary leads with the measured findings when present, and reports review counts as raw totals over the window rather than as a projection. +- The `report` and `onboard` recipes stop asking for fix counts, give one method for counting `surfaced`, and require the agent's own recap to keep "found in the code today" apart from "raised in review over the window". + +Delivery shape: **single PR.** The schema change, the measurement, and the recipe changes are only correct together: landing the schema alone would publish reports with no findings, and landing the recipes alone would describe fields the CLI cannot send. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `cli-report`: the draft and the payload carry CLI-measured findings; the facts file loses fix counts and renames origin to surfaced; rules carry a state; the CLI stops computing monthly rates; the summary and the recipe change to match. + +## Impact + +- `packages/cli/src/commands/check.ts`: the scan it performs is extracted so `report` can run the same one. +- `packages/cli/src/report/` (`draft.ts`, `assemble.ts`, `types.ts`, `summary.ts`) and `packages/cli/src/schemas/report-facts.ts`. +- `packages/cli/src/generated/report-v1.schema.json`, refreshed by `scripts/fetch-report-schema.ts`. +- `packages/cli/src/agent/report.md` and `packages/cli/src/agent/onboard.md`. +- Tests under `packages/cli/test/` for report and check. +- No change to `taskless check`'s output or exit codes. +- Ships in a nightly. `taskless report` itself has not been released, so this extends its pending `patch` changeset rather than adding a second release note for a command no user has seen. diff --git a/openspec/changes/archive/2026-10-14-report-measured-findings/specs/cli-report/spec.md b/openspec/changes/archive/2026-10-14-report-measured-findings/specs/cli-report/spec.md new file mode 100644 index 00000000..f03b14a2 --- /dev/null +++ b/openspec/changes/archive/2026-10-14-report-measured-findings/specs/cli-report/spec.md @@ -0,0 +1,197 @@ +## ADDED Requirements + +### Requirement: The CLI measures what the rules find today + +The CLI SHALL measure, itself, how often each written rule matches the repository's code, by running the project's static rules over the whole project as `taskless check --anonymous` would. The measurement SHALL perform no network call and SHALL NOT change `taskless check`'s own output or exit code. For each rule under `.taskless/rules/` it SHALL record `violations`, the number of results attributed to that rule, and `files`, the number of distinct files among those results. It SHALL record `findings.violations`, the sum of every rule's `violations`, and `findings.files`, the number of distinct files with at least one result of any rule, so a file that breaks several rules counts once. File identity SHALL be the path relative to the repository root. + +The counts SHALL be all-or-nothing. The CLI SHALL record them only when every written rule in the report is one it ran and every result it collected is attributed to one of them. When any written rule is a `legacy` rule, or a rule under `.taskless/rules/runtime/`, or when the scan reports an engine or plan failure, or when a result names a rule id that no written rule has, the CLI SHALL record no `findings` and no rule SHALL carry `violations` or `files`. The facts file SHALL NOT supply any of these values. + +#### Scenario: Every written rule is measured + +- **WHEN** `.taskless/rules/` holds three `sg` rules and one `vale` rule, the facts list no `legacy` rules, and the scan finds 4, 75, 0 and 12 results for them +- **THEN** each of the four rules SHALL carry its own `violations` and `files` +- **AND** `findings.violations` SHALL be 91 + +#### Scenario: A file broken by two rules counts once + +- **WHEN** one file holds results from two different rules, and no other file holds results +- **THEN** each of the two rules SHALL carry `files: 1` +- **AND** `findings.files` SHALL be 1 + +#### Scenario: A rule with no results is measured as zero + +- **WHEN** a written rule under `.taskless/rules/sg/` matches nothing and every other condition for counts holds +- **THEN** that rule SHALL carry `violations: 0` and `files: 0` + +#### Scenario: A legacy rule omits every count + +- **WHEN** the facts list a `legacy` rule +- **THEN** the report SHALL carry no `findings` +- **AND** no rule SHALL carry `violations` or `files` + +#### Scenario: A local runtime rule omits every count + +- **WHEN** `.taskless/rules/runtime/` holds a rule +- **THEN** the report SHALL carry no `findings` +- **AND** no rule SHALL carry `violations` or `files` + +#### Scenario: An engine failure omits every count + +- **WHEN** an engine fails during the scan +- **THEN** the report SHALL carry no `findings` +- **AND** no rule SHALL carry `violations` or `files` + +#### Scenario: A result the CLI cannot attribute omits every count + +- **WHEN** the scan returns a result whose rule id names no rule in the report +- **THEN** the report SHALL carry no `findings` +- **AND** no rule SHALL carry `violations` or `files` + +#### Scenario: Candidates never carry counts + +- **WHEN** the facts list a `cloud` candidate and every written rule is measured +- **THEN** the candidate SHALL carry no `violations` or `files` +- **AND** `findings` SHALL count only the written rules + +#### Scenario: The facts cannot supply counts + +- **WHEN** the facts file gives a rule `violations` or `files`, or gives a top-level `findings` +- **THEN** the command SHALL refuse the field as unknown under `REPORT_INVALID` + +### Requirement: The payload carries projection inputs and no rates + +The CLI SHALL NOT send monthly rates. tskl.es derives them from each rule's `surfaced` counts and the window. The payload's `projection` SHALL carry only `inputs`: the agent's estimates for `contextTokens` and `callsPerCatch`, and the CLI's defaults for every other input. The facts file SHALL NOT accept rates. + +#### Scenario: Rates in the facts are not accepted + +- **WHEN** the facts file includes a `perMonth` value +- **THEN** the command SHALL refuse it as an unknown field under `REPORT_INVALID` + +#### Scenario: The payload sends inputs only + +- **WHEN** a report publishes +- **THEN** its `projection` SHALL hold `inputs` and nothing else + +## MODIFIED Requirements + +### Requirement: The draft states what the CLI knows and what the agent must supply + +`taskless report` with no mode flag SHALL print a single JSON object to stdout and exit 0. The object SHALL contain every report field the CLI can establish from the working directory: the producer (`kind: "cli"` and this build's version), the scope derived from the repository's `origin` remote, one entry per rule directory under `.taskless/rules/` with its id, engine, tier, language, and test status and fixtures, the default projection inputs, and what the rules find in the code today as the CLI measured it. It SHALL also contain `missing`, a list naming each fact the agent must supply before the report can publish. A fact the CLI can establish SHALL NOT appear in `missing`. + +The draft SHALL NOT perform any network call. + +#### Scenario: A repository with local rules + +- **WHEN** `.taskless/rules/` holds three `sg` rules and one `vale` rule and the user runs `taskless report` +- **THEN** stdout SHALL be a JSON object listing four rules, each with `tier: "cli"`, its engine, and its test status +- **AND** each rule SHALL carry its measured `violations` and `files`, and the draft SHALL carry `findings` +- **AND** `missing` SHALL name the session facts still required, including `history` and the projection estimates + +#### Scenario: A GitHub origin becomes a GitHub scope + +- **WHEN** the repository's `origin` is a GitHub remote +- **THEN** the draft's scope SHALL carry `provider: "github"` with the owner, repository, and canonical URL + +#### Scenario: Any other origin becomes an unnamed scope + +- **WHEN** the repository has no `origin`, or a non-GitHub `origin` +- **THEN** the draft's scope SHALL carry `provider: "none"` with a label + +#### Scenario: The draft is offline + +- **WHEN** `taskless report` runs with no network available +- **THEN** it SHALL print the draft and exit 0 + +### Requirement: The agent supplies session facts, not a report + +`taskless report --from ` SHALL read a facts file, not a ReportV1. The facts file SHALL carry only what the agent knows from its session: the history counts and the window they were mined over, per-rule titles and `surfaced` counts (the merged pull requests in the window whose review raised the rule's pattern, and the review comments that asked for it), rules authored into an existing linter (`legacy` tier), runtime-shaped candidates the agent did not author (`cloud` tier), the agent's estimates of context tokens per session and calls per catch, the agent's own redaction counts, and the fixtures it elects to withhold. The CLI SHALL merge the facts with what it establishes itself, and SHALL treat its own values as authoritative for every field it can establish. + +Every published rule SHALL carry a `state`. Rules under `.taskless/rules/` and `legacy` rules SHALL be `written`; `cloud` candidates from the facts file SHALL be `candidate`. + +The facts file SHALL be validated against a published facts schema before anything else happens. The `report` recipe SHALL embed that schema. + +#### Scenario: Facts cannot set what the CLI knows + +- **WHEN** the facts file gives a rule under `.taskless/rules/` an engine, a language, a test status, or a state +- **THEN** the command SHALL refuse the field as unknown under `REPORT_INVALID` +- **AND** a report that publishes SHALL carry the engine, language, test status, and state the CLI established + +#### Scenario: Malformed facts are refused before any other work + +- **WHEN** the facts file is not valid JSON or fails the facts schema +- **THEN** the command SHALL exit non-zero with `REPORT_INVALID` naming each failing field +- **AND** SHALL NOT perform any network call + +#### Scenario: Cloud candidates are carried as cloud tier + +- **WHEN** the facts file lists a runtime-shaped candidate +- **THEN** the published report SHALL include it with `tier: "cloud"`, `engine: "runtime"`, and `state: "candidate"` +- **AND** its tests status SHALL be `not-run` + +#### Scenario: Fix counts are not accepted + +- **WHEN** the facts file gives a rule `fixed`, `fixedBasis`, or `origin` +- **THEN** the command SHALL refuse the field as unknown under `REPORT_INVALID` + +### Requirement: A published report is recorded and summarized + +On a 201 response the CLI SHALL write `.taskless/reports/.json` (UTC) holding the report URL, the delete credential exactly as the service returned it, and the expiry. It SHALL then print a brief summary rendered from a CLI template: one line per rule giving its title and tier; when the report carries `findings`, a line stating how many problems the rules found in how many files in the code today; a line stating how many review comments across how many pull requests raised the rules' patterns over the window; the report URL; the expiry; and one sentence stating the report can be deleted after review. The summary SHALL NOT present a monthly rate. With `--json` it SHALL instead print a JSON object carrying the URL, the expiry, the record's path, and the rendered summary. The delete credential SHALL NOT appear on stdout in either form. + +#### Scenario: The record is written + +- **WHEN** a publish succeeds +- **THEN** exactly one new file SHALL exist under `.taskless/reports/` holding the URL, the delete credential, and the expiry + +#### Scenario: The summary names the link and the way back + +- **WHEN** a publish succeeds without `--json` +- **THEN** stdout SHALL list each published rule's title and tier +- **AND** SHALL print the report URL +- **AND** SHALL state that the report can be deleted after review + +#### Scenario: The summary leads with what the rules found + +- **WHEN** a publish succeeds and the report carries `findings` of 342 violations in 118 files +- **THEN** the summary SHALL state that the rules found 342 problems in 118 files in the code today + +#### Scenario: The summary omits findings it does not have + +- **WHEN** a publish succeeds and the report carries no `findings` +- **THEN** the summary SHALL NOT state a count of problems found + +#### Scenario: The delete credential stays on disk + +- **WHEN** a publish succeeds, with or without `--json` +- **THEN** stdout SHALL NOT contain the delete credential + +### Requirement: The report recipe drives the flow + +The CLI SHALL embed a `report` recipe following the canonical recipe template. It SHALL instruct the agent to: run `taskless report` and read `missing`; supply each missing fact from its session, writing the facts to `.taskless/.tmp-report.json`; count each rule's `surfaced` by the method the recipe states; estimate context tokens and calls per catch from how this user's sessions actually run; run `taskless report --from` that path; on `REPORT_INSUFFICIENT`, ask the user only what would let it author the rules or establish the facts the refusal names, then retry; on success, show the CLI's summary as printed and tell the user the report can be deleted after review if they ask; delete the facts file; and on a request to delete, run `taskless report --delete` with the record. The recipe SHALL embed the facts schema and SHALL state that tskl.es does not redact, so anything the agent cannot vouch for is to be withheld. + +The counting method SHALL be a single procedure: a pull request counts toward a rule's `surfaced.prs` when it merged inside the window and at least one review comment on it asked for the change the rule enforces; a comment counts toward `surfaced.reviewComments` when a reader of that comment would say it asks for that change. Every author counts: a comment from a bot account or from the pull request's own author SHALL count the same as a reviewer's, since each one spends cycles, tokens, time, or money. The recipe SHALL direct the agent to judge each comment by reading it, not by matching keywords. + +Whenever the agent restates the report's numbers in its own words, the recipe SHALL require it to keep counts the CLI measured in the code today apart from counts mined from review history over the window, and to say which is which. + +#### Scenario: The recipe asks questions only on refusal + +- **WHEN** the `report` recipe's steps are read +- **THEN** the only step that permits a question to the user SHALL be the one handling `REPORT_INSUFFICIENT` + +#### Scenario: The recipe embeds the facts schema + +- **WHEN** an agent runs `taskless agent report` +- **THEN** stdout SHALL open with `# Topic: report` and contain the JSON Schema for the facts file + +#### Scenario: The recipe states one counting method + +- **WHEN** an agent runs `taskless agent report` +- **THEN** stdout SHALL state how to count `surfaced.prs` and `surfaced.reviewComments`, including that bot and author comments count +- **AND** SHALL NOT ask for a count of fixes + +## REMOVED Requirements + +### Requirement: The CLI computes the projection + +**Reason**: tskl.es now derives both monthly rates from each rule's `surfaced` counts and the window, and refuses a payload that carries `projection.perMonth`. The rates were also computed from `fixed`, which the schema removed. + +**Migration**: Replaced by "The payload carries projection inputs and no rates". The facts file still supplies `window`, `surfaced`, and `estimates`; nothing the agent writes changes beyond dropping `fixed` and `fixedBasis`. diff --git a/openspec/changes/archive/2026-10-14-report-measured-findings/tasks.md b/openspec/changes/archive/2026-10-14-report-measured-findings/tasks.md new file mode 100644 index 00000000..019a0471 --- /dev/null +++ b/openspec/changes/archive/2026-10-14-report-measured-findings/tasks.md @@ -0,0 +1,28 @@ +## 1. Extract the scan from `check` + +- [x] 1.1 Move the run-directory, `planCheck`, engine-config assembly, and `runEngines` steps of `packages/cli/src/commands/check.ts` into a function under `packages/cli/src/rules/` that returns the plan, the dispatch, and the run's relative path, and have `check` call it; verify `pnpm --filter @taskless/cli test check` (check, check-rule-filter, check-snapshot, check-gitignore, mixed-engine-check, runtime-check) passes unchanged +- [x] 1.2 Confirm the extracted function with `anonymous: true` makes no network call by adding a test that runs it with fetch stubbed to throw over a fixture with `sg`, `vale`, and `runtime` rules, and asserts runtime rules are skipped and sg/vale results return + +## 2. Measure findings for the report + +- [x] 2.1 Refresh `packages/cli/src/generated/report-v1.schema.json` with `pnpm --filter @taskless/cli generate:report-schema` and update `ReportV1` in `packages/cli/src/report/types.ts` (add `findings`, `state`, `violations`, `files`, `surfaced`; drop `fixed`, `fixedBasis`, `origin`, `perMonth`, `PerTier`); verify `pnpm typecheck` reports only the call sites groups 2 and 3 fix +- [x] 2.2 Add the all-or-nothing counting function in `packages/cli/src/report/` (per-rule `violations`/`files`, distinct `findings.files` by repo-relative path, omitted on legacy, local runtime, any plan or dispatch failure, or an unattributed `ruleId`); verify with unit tests covering every scenario under "The CLI measures what the rules find today" +- [x] 2.3 Run the scan from `buildDraft` and carry the counts on each local rule and as draft-level `findings`; verify a `report.test.ts` case over a fixture repo with known violations prints them in the draft + +## 3. Facts, assembly, and summary + +- [x] 3.1 In `packages/cli/src/schemas/report-facts.ts`, drop `fixed` and `fixedBasis`, rename `origin` to `surfaced` with `{ prs, reviewComments }` and descriptions matching the recipe's counting method; verify a test that facts carrying `fixed`, `fixedBasis`, `origin`, `violations`, `findings`, or `perMonth` are refused with `REPORT_INVALID` +- [x] 3.2 In `packages/cli/src/report/assemble.ts`, set `state` per tier, carry `surfaced`, re-apply the counting function after merging facts (a `legacy` entry removes the counts), and send `projection: { inputs }` only; verify every assembled payload in `report.test.ts` passes `reportSchemaIssues` against the refreshed schema, including one with findings and one with a legacy rule and none +- [x] 3.3 Rewrite `packages/cli/src/report/summary.ts` to print the findings line when present and the raw `surfaced` totals over the window, with no monthly rate; verify summary tests for the with-findings and without-findings scenarios +- [x] 3.4 Update `report-telemetry.test.ts` and any other report test that built `fixed`/`origin`/`perMonth` fixtures; verify `pnpm --filter @taskless/cli test report` passes + +## 4. Recipes + +- [x] 4.1 Rewrite `packages/cli/src/agent/report.md` step 2 to drop fix counting, state the single `surfaced` counting method (merged in window, read each comment, bots and the PR author count), note that the draft's `findings` are measured by the CLI and cannot be supplied, and require the agent's own recap to label "in the code today" apart from "raised in review over the window"; bump its topic version; verify `pnpm --filter @taskless/cli test prompts recipe` passes +- [x] 4.2 Update `packages/cli/src/agent/onboard.md` step 6 to note `surfaced` counts by the report recipe's method instead of how often a pattern was fixed; bump its topic version; verify the same prompt tests pass + +## 5. Release and checks + +- [x] 5.1 Extend the pending `streamlined-onboarding-report` changeset, which introduces `taskless report` and has not been released, to describe the measured findings instead of projections; verify it no longer mentions projected catches +- [x] 5.2 Run `pnpm typecheck` and `pnpm lint` and verify both pass +- [x] 5.3 Build with `pnpm build:next`, run `pnpm cli report` in a scratch repo with known violations, and verify the draft's `findings` match `pnpm cli check --json --anonymous` for the same tree diff --git a/openspec/specs/cli-report/spec.md b/openspec/specs/cli-report/spec.md index 1da3cc86..0361d804 100644 --- a/openspec/specs/cli-report/spec.md +++ b/openspec/specs/cli-report/spec.md @@ -22,7 +22,7 @@ The CLI SHALL provide a `taskless report` subcommand accepting the standard `--d ### Requirement: The draft states what the CLI knows and what the agent must supply -`taskless report` with no mode flag SHALL print a single JSON object to stdout and exit 0. The object SHALL contain every report field the CLI can establish from the working directory: the producer (`kind: "cli"` and this build's version), the scope derived from the repository's `origin` remote, one entry per rule directory under `.taskless/rules/` with its id, engine, tier, language, and test status and fixtures, and the default projection inputs. It SHALL also contain `missing`, a list naming each fact the agent must supply before the report can publish. A fact the CLI can establish SHALL NOT appear in `missing`. +`taskless report` with no mode flag SHALL print a single JSON object to stdout and exit 0. The object SHALL contain every report field the CLI can establish from the working directory: the producer (`kind: "cli"` and this build's version), the scope derived from the repository's `origin` remote, one entry per rule directory under `.taskless/rules/` with its id, engine, tier, language, and test status and fixtures, the default projection inputs, and what the rules find in the code today as the CLI measured it. It SHALL also contain `missing`, a list naming each fact the agent must supply before the report can publish. A fact the CLI can establish SHALL NOT appear in `missing`. The draft SHALL NOT perform any network call. @@ -30,6 +30,7 @@ The draft SHALL NOT perform any network call. - **WHEN** `.taskless/rules/` holds three `sg` rules and one `vale` rule and the user runs `taskless report` - **THEN** stdout SHALL be a JSON object listing four rules, each with `tier: "cli"`, its engine, and its test status +- **AND** each rule SHALL carry its measured `violations` and `files`, and the draft SHALL carry `findings` - **AND** `missing` SHALL name the session facts still required, including `history` and the projection estimates #### Scenario: A GitHub origin becomes a GitHub scope @@ -49,15 +50,17 @@ The draft SHALL NOT perform any network call. ### Requirement: The agent supplies session facts, not a report -`taskless report --from ` SHALL read a facts file, not a ReportV1. The facts file SHALL carry only what the agent knows from its session: the history counts and the window they were mined over, per-rule titles and origin counts and fix counts with their basis, rules authored into an existing linter (`legacy` tier), runtime-shaped candidates the agent did not author (`cloud` tier), the agent's estimates of context tokens per session and calls per catch, the agent's own redaction counts, and the fixtures it elects to withhold. The CLI SHALL merge the facts with what it establishes itself, and SHALL treat its own values as authoritative for every field it can establish. +`taskless report --from ` SHALL read a facts file, not a ReportV1. The facts file SHALL carry only what the agent knows from its session: the history counts and the window they were mined over, per-rule titles and `surfaced` counts (the merged pull requests in the window whose review raised the rule's pattern, and the review comments that asked for it), rules authored into an existing linter (`legacy` tier), runtime-shaped candidates the agent did not author (`cloud` tier), the agent's estimates of context tokens per session and calls per catch, the agent's own redaction counts, and the fixtures it elects to withhold. The CLI SHALL merge the facts with what it establishes itself, and SHALL treat its own values as authoritative for every field it can establish. + +Every published rule SHALL carry a `state`. Rules under `.taskless/rules/` and `legacy` rules SHALL be `written`; `cloud` candidates from the facts file SHALL be `candidate`. The facts file SHALL be validated against a published facts schema before anything else happens. The `report` recipe SHALL embed that schema. #### Scenario: Facts cannot set what the CLI knows -- **WHEN** the facts file gives a rule under `.taskless/rules/` an engine, a language, or a test status +- **WHEN** the facts file gives a rule under `.taskless/rules/` an engine, a language, a test status, or a state - **THEN** the command SHALL refuse the field as unknown under `REPORT_INVALID` -- **AND** a report that publishes SHALL carry the engine, language, and test status the CLI established +- **AND** a report that publishes SHALL carry the engine, language, test status, and state the CLI established #### Scenario: Malformed facts are refused before any other work @@ -68,22 +71,13 @@ The facts file SHALL be validated against a published facts schema before anythi #### Scenario: Cloud candidates are carried as cloud tier - **WHEN** the facts file lists a runtime-shaped candidate -- **THEN** the published report SHALL include it with `tier: "cloud"` and `engine: "runtime"` +- **THEN** the published report SHALL include it with `tier: "cloud"`, `engine: "runtime"`, and `state: "candidate"` - **AND** its tests status SHALL be `not-run` -### Requirement: The CLI computes the projection - -The CLI SHALL compute `projection.perMonth` itself from the merged facts and SHALL NOT accept per-month figures from the facts file. Per-tier counts SHALL be normalized to a 30-day month over the mining window the facts declare. The projection's `inputs` SHALL carry the agent's estimates for `contextTokens` and `callsPerCatch` and the CLI's defaults for every other input. - -#### Scenario: Per-month figures in the facts are not accepted - -- **WHEN** the facts file includes a `perMonth` value -- **THEN** the command SHALL refuse it as an unknown field under `REPORT_INVALID` - -#### Scenario: Window normalization +#### Scenario: Fix counts are not accepted -- **WHEN** the facts declare a 90-day window and rules in the `cli` tier with a combined fix count of 30 -- **THEN** the published `projection.perMonth.agentCatches.cli` SHALL be 10 +- **WHEN** the facts file gives a rule `fixed`, `fixedBasis`, or `origin` +- **THEN** the command SHALL refuse the field as unknown under `REPORT_INVALID` ### Requirement: Publishing is gated on a meaningful report @@ -177,7 +171,7 @@ The CLI SHALL publish with `POST https://tskl.es/api/v1/reports` and SHALL alway ### Requirement: A published report is recorded and summarized -On a 201 response the CLI SHALL write `.taskless/reports/.json` (UTC) holding the report URL, the delete credential exactly as the service returned it, and the expiry. It SHALL then print a brief summary rendered from a CLI template: one line per rule giving its title and tier, the headline projection, the report URL, the expiry, and one sentence stating the report can be deleted after review. With `--json` it SHALL instead print a JSON object carrying the URL, the expiry, the record's path, and the rendered summary. The delete credential SHALL NOT appear on stdout in either form. +On a 201 response the CLI SHALL write `.taskless/reports/.json` (UTC) holding the report URL, the delete credential exactly as the service returned it, and the expiry. It SHALL then print a brief summary rendered from a CLI template: one line per rule giving its title and tier; when the report carries `findings`, a line stating how many problems the rules found in how many files in the code today; a line stating how many review comments across how many pull requests raised the rules' patterns over the window; the report URL; the expiry; and one sentence stating the report can be deleted after review. The summary SHALL NOT present a monthly rate. With `--json` it SHALL instead print a JSON object carrying the URL, the expiry, the record's path, and the rendered summary. The delete credential SHALL NOT appear on stdout in either form. #### Scenario: The record is written @@ -191,6 +185,16 @@ On a 201 response the CLI SHALL write `.taskless/reports/.jso - **AND** SHALL print the report URL - **AND** SHALL state that the report can be deleted after review +#### Scenario: The summary leads with what the rules found + +- **WHEN** a publish succeeds and the report carries `findings` of 342 violations in 118 files +- **THEN** the summary SHALL state that the rules found 342 problems in 118 files in the code today + +#### Scenario: The summary omits findings it does not have + +- **WHEN** a publish succeeds and the report carries no `findings` +- **THEN** the summary SHALL NOT state a count of problems found + #### Scenario: The delete credential stays on disk - **WHEN** a publish succeeds, with or without `--json` @@ -227,7 +231,11 @@ On a 201 response the CLI SHALL write `.taskless/reports/.jso ### Requirement: The report recipe drives the flow -The CLI SHALL embed a `report` recipe following the canonical recipe template. It SHALL instruct the agent to: run `taskless report` and read `missing`; supply each missing fact from its session, writing the facts to `.taskless/.tmp-report.json`; estimate context tokens and calls per catch from how this user's sessions actually run; run `taskless report --from` that path; on `REPORT_INSUFFICIENT`, ask the user only what would let it author the rules or establish the facts the refusal names, then retry; on success, show the CLI's summary as printed and tell the user the report can be deleted after review if they ask; delete the facts file; and on a request to delete, run `taskless report --delete` with the record. The recipe SHALL embed the facts schema and SHALL state that tskl.es does not redact, so anything the agent cannot vouch for is to be withheld. +The CLI SHALL embed a `report` recipe following the canonical recipe template. It SHALL instruct the agent to: run `taskless report` and read `missing`; supply each missing fact from its session, writing the facts to `.taskless/.tmp-report.json`; count each rule's `surfaced` by the method the recipe states; estimate context tokens and calls per catch from how this user's sessions actually run; run `taskless report --from` that path; on `REPORT_INSUFFICIENT`, ask the user only what would let it author the rules or establish the facts the refusal names, then retry; on success, show the CLI's summary as printed and tell the user the report can be deleted after review if they ask; delete the facts file; and on a request to delete, run `taskless report --delete` with the record. The recipe SHALL embed the facts schema and SHALL state that tskl.es does not redact, so anything the agent cannot vouch for is to be withheld. + +The counting method SHALL be a single procedure: a pull request counts toward a rule's `surfaced.prs` when it merged inside the window and at least one review comment on it asked for the change the rule enforces; a comment counts toward `surfaced.reviewComments` when a reader of that comment would say it asks for that change. Every author counts: a comment from a bot account or from the pull request's own author SHALL count the same as a reviewer's, since each one spends cycles, tokens, time, or money. The recipe SHALL direct the agent to judge each comment by reading it, not by matching keywords. + +Whenever the agent restates the report's numbers in its own words, the recipe SHALL require it to keep counts the CLI measured in the code today apart from counts mined from review history over the window, and to say which is which. #### Scenario: The recipe asks questions only on refusal @@ -239,6 +247,12 @@ The CLI SHALL embed a `report` recipe following the canonical recipe template. I - **WHEN** an agent runs `taskless agent report` - **THEN** stdout SHALL open with `# Topic: report` and contain the JSON Schema for the facts file +#### Scenario: The recipe states one counting method + +- **WHEN** an agent runs `taskless agent report` +- **THEN** stdout SHALL state how to count `surfaced.prs` and `surfaced.reviewComments`, including that bot and author comments count +- **AND** SHALL NOT ask for a count of fixes + ### Requirement: Report emits intent telemetry without report content The `report` subcommand SHALL emit `cli_report_drafted` for a draft, `cli_report_published` on a successful publish carrying only rule counts by tier and the gate's outcome, `cli_report_refused` carrying the refusal code, and `cli_report_deleted` on a successful delete. No event SHALL carry the report URL, the delete credential, any rule title, or any fixture. @@ -248,3 +262,75 @@ The `report` subcommand SHALL emit `cli_report_drafted` for a draft, `cli_report - **WHEN** a report publishes - **THEN** PostHog SHALL receive `cli_report_published` with per-tier rule counts - **AND** the event SHALL NOT contain the report URL or the delete credential + +### Requirement: The CLI measures what the rules find today + +The CLI SHALL measure, itself, how often each written rule matches the repository's code, by running the project's static rules over the whole project as `taskless check --anonymous` would. The measurement SHALL perform no network call and SHALL NOT change `taskless check`'s own output or exit code. For each rule under `.taskless/rules/` it SHALL record `violations`, the number of results attributed to that rule, and `files`, the number of distinct files among those results. It SHALL record `findings.violations`, the sum of every rule's `violations`, and `findings.files`, the number of distinct files with at least one result of any rule, so a file that breaks several rules counts once. File identity SHALL be the path relative to the repository root. + +The counts SHALL be all-or-nothing. The CLI SHALL record them only when every written rule in the report is one it ran and every result it collected is attributed to one of them. When any written rule is a `legacy` rule, or a rule under `.taskless/rules/runtime/`, or when the scan reports an engine or plan failure, or when a result names a rule id that no written rule has, the CLI SHALL record no `findings` and no rule SHALL carry `violations` or `files`. The facts file SHALL NOT supply any of these values. + +#### Scenario: Every written rule is measured + +- **WHEN** `.taskless/rules/` holds three `sg` rules and one `vale` rule, the facts list no `legacy` rules, and the scan finds 4, 75, 0 and 12 results for them +- **THEN** each of the four rules SHALL carry its own `violations` and `files` +- **AND** `findings.violations` SHALL be 91 + +#### Scenario: A file broken by two rules counts once + +- **WHEN** one file holds results from two different rules, and no other file holds results +- **THEN** each of the two rules SHALL carry `files: 1` +- **AND** `findings.files` SHALL be 1 + +#### Scenario: A rule with no results is measured as zero + +- **WHEN** a written rule under `.taskless/rules/sg/` matches nothing and every other condition for counts holds +- **THEN** that rule SHALL carry `violations: 0` and `files: 0` + +#### Scenario: A legacy rule omits every count + +- **WHEN** the facts list a `legacy` rule +- **THEN** the report SHALL carry no `findings` +- **AND** no rule SHALL carry `violations` or `files` + +#### Scenario: A local runtime rule omits every count + +- **WHEN** `.taskless/rules/runtime/` holds a rule +- **THEN** the report SHALL carry no `findings` +- **AND** no rule SHALL carry `violations` or `files` + +#### Scenario: An engine failure omits every count + +- **WHEN** an engine fails during the scan +- **THEN** the report SHALL carry no `findings` +- **AND** no rule SHALL carry `violations` or `files` + +#### Scenario: A result the CLI cannot attribute omits every count + +- **WHEN** the scan returns a result whose rule id names no rule in the report +- **THEN** the report SHALL carry no `findings` +- **AND** no rule SHALL carry `violations` or `files` + +#### Scenario: Candidates never carry counts + +- **WHEN** the facts list a `cloud` candidate and every written rule is measured +- **THEN** the candidate SHALL carry no `violations` or `files` +- **AND** `findings` SHALL count only the written rules + +#### Scenario: The facts cannot supply counts + +- **WHEN** the facts file gives a rule `violations` or `files`, or gives a top-level `findings` +- **THEN** the command SHALL refuse the field as unknown under `REPORT_INVALID` + +### Requirement: The payload carries projection inputs and no rates + +The CLI SHALL NOT send monthly rates. tskl.es derives them from each rule's `surfaced` counts and the window. The payload's `projection` SHALL carry only `inputs`: the agent's estimates for `contextTokens` and `callsPerCatch`, and the CLI's defaults for every other input. The facts file SHALL NOT accept rates. + +#### Scenario: Rates in the facts are not accepted + +- **WHEN** the facts file includes a `perMonth` value +- **THEN** the command SHALL refuse it as an unknown field under `REPORT_INVALID` + +#### Scenario: The payload sends inputs only + +- **WHEN** a report publishes +- **THEN** its `projection` SHALL hold `inputs` and nothing else diff --git a/packages/cli/src/agent/onboard.md b/packages/cli/src/agent/onboard.md index 808eef66..56d041a6 100644 --- a/packages/cli/src/agent/onboard.md +++ b/packages/cli/src/agent/onboard.md @@ -1,4 +1,4 @@ -# Topic: onboard (CLI v%(CLI_VERSION)s / topic v6) +# Topic: onboard (CLI v%(CLI_VERSION)s / topic v7) ## Goal Take a user who has just installed Taskless from zero rules to a @@ -118,8 +118,11 @@ making the case for running them in CI. `sg`, `vale`, or `runtime`. `route` decides for real when you write the rule, and may land it somewhere else. Three high-quality candidates beat ten speculative ones. Note, for the report, how - many review comments and pull requests each one came from and how - often the pattern was fixed in the window you mined. + many pull requests and review comments raised each one in the + window you mined: a pull request counts when it merged in the + window and a review comment on it asked for the change, and each + such comment counts. Bot and author comments count too. Judge each + comment by reading it, never by matching keywords. 7. **Write the rules.** For every candidate whose destination is `legacy`, `sg`, or `vale`, follow the `%(TASKLESS_CLI)s agent route` diff --git a/packages/cli/src/agent/report.md b/packages/cli/src/agent/report.md index a5bc41cc..929ff658 100644 --- a/packages/cli/src/agent/report.md +++ b/packages/cli/src/agent/report.md @@ -1,4 +1,4 @@ -# Topic: report (CLI v%(CLI_VERSION)s / topic v1) +# Topic: report (CLI v%(CLI_VERSION)s / topic v2) ## Goal Publish a Taskless report: an unlisted page on tskl.es that shows what @@ -31,6 +31,12 @@ later and deletes one on request. fixtures, and `missing`, the facts only you can supply. Read `missing`; it is the checklist for the next step. + The draft also carries what the rules find in the code today: + `violations` and `files` on each rule, and `findings` across all of + them, counted by the CLI's own scan. You cannot supply or change + these. When they are absent, `unmeasured` says why, and the report + publishes without them. + 2. **Supply what is missing.** Write `.taskless/.tmp-report.json` following the schema below. Every entry comes from this session or from sources you can reach now, never @@ -42,13 +48,13 @@ later and deletes one on request. recurring review patterns were rule-shaped versus judgment calls. Zeros are honest when there was nothing to mine. - `rules`: one entry per rule the draft listed, with a title a - reader understands, how many review comments and pull requests - asked for what it enforces, and how often the pattern was fixed in - the window. Set `fixedBasis` to `measured` only when you counted. + reader understands and its `surfaced` counts, counted as described + under `## Counting surfaced`. - `legacy`: rules you wrote into the repository's own linter, which the CLI cannot see. - `cloud`: runtime-shaped candidates you judged better solved by a - runtime rule and did not author. + runtime rule and did not author. Count their `surfaced` the same + way. - `estimates`: `contextTokens` is how large this user's sessions typically run; `callsPerCatch` is how many extra agent turns a missed issue costs them before it is fixed. Estimate from what you @@ -73,10 +79,17 @@ later and deletes one on request. asking. 4. **Show the result.** On success the CLI prints a short summary: the - rules, the projection, the link, and when it expires. Show it to the - user exactly as printed. Then tell them, in one line, that the - report can be deleted after they have reviewed it, and that they - only need to ask. + rules, what they found in the code today, what review raised over + the window, the link, and when it expires. Show it to the user + exactly as printed. Then tell them, in one line, that the report can + be deleted after they have reviewed it, and that they only need to + ask. + + If you restate any of the report's numbers in your own words, keep + the two kinds apart and label each: counts the CLI found **in the + code today**, and counts mined **from review over the window**. A + rule with 75 violations today that was raised in 23 pull requests + has two different numbers, and neither is the other. 5. **Clean up.** Delete `.taskless/.tmp-report.json`. The CLI kept a record of the report in `.taskless/reports/`, which is gitignored @@ -94,6 +107,26 @@ later and deletes one on request. confirms the deletion. Tell the user cached copies of the page may be served for up to a day afterwards. +## Counting surfaced + +Every agent counts `surfaced` the same way, so reports from different +sessions compare: + +- **`surfaced.prs`**: pull or merge requests merged inside `window` + with at least one review comment asking for the change the rule + enforces. +- **`surfaced.reviewComments`**: the review comments on those pull + requests that ask for that change. Count inline review comments and + review bodies. +- **Every author counts.** A comment from a bot account or from the + pull request's own author counts the same as a reviewer's: each one + spent cycles, tokens, time, or money. +- **Judge each comment by reading it.** A comment counts when a reader + would say it asks for the change the rule enforces. Do not match + keywords: "fixed" or "removed" in a reply says nothing about which + rule it concerns. +- A comment that asks for two rules' changes counts once for each. + ## Facts schema The `--from` JSON file conforms to: diff --git a/packages/cli/src/commands/check.ts b/packages/cli/src/commands/check.ts index cdb619d0..94a83e8e 100644 --- a/packages/cli/src/commands/check.ts +++ b/packages/cli/src/commands/check.ts @@ -10,8 +10,7 @@ import { realpath, stat } from "node:fs/promises"; import process from "node:process"; import { defineCommand } from "citty"; -import { hasValeRules, runEngines } from "../rules/dispatch"; -import { assembleEngineConfigs } from "../rules/assemble"; +import { hasValeRules } from "../rules/dispatch"; import { splitRawArguments } from "../util/argv"; import { formatText } from "../util/format"; import { listRuleIds, planEngineDispatch } from "../rules/engines"; @@ -22,9 +21,7 @@ import { CLIError } from "../util/cli-error"; import { requireCurrentSchema } from "../filesystem/migrate"; import { discoverRuntimeRules } from "../rules/runtime/discover"; import { resolveRuleSelection, type RuleSelection } from "../rules/rule-filter"; -import { planCheck } from "../rules/plan-check"; -import { openRun } from "../rules/run-directory"; -import { fromProjectRoot } from "../rules/snapshot"; +import { scanProject } from "../rules/scan-project"; import { markNotice } from "../util/notices"; import { getCliPrefix } from "../util/package-manager"; @@ -452,71 +449,24 @@ export const checkCommand = defineCommand({ return; } - // This run's own directory under `.taskless/.run/`: the snapshot, the - // assembled configs, and the logs. Removed when the run ends, however - // it ends, unless `--preserve-logs` keeps it for debugging. const preserve = Boolean(args["preserve-logs"]); - const run = await openRun(cwd, { preserve }); try { - // Planned before dispatch, not during it: planning snapshots the rules - // tree and consults auth and reconcile state, which decides WHAT runs. - // Everything below reads the snapshot, never `.taskless/rules/`, so - // the bytes that run are the bytes that were judged. - const plan = await planCheck(cwd, run, { + const { plan, dispatched, runDirectory } = await scanProject(cwd, { + paths: existingPaths, anonymous: args.anonymous, dangerouslyRunScripts: Boolean(args["dangerously-run-scripts"]), - }); - for (const notice of plan.notices) warnNotice(notice); - for (const failure of plan.failures) warn(`Error: ${failure}`); - for (const skipped of plan.skipped) { - warnNotice( - `runtime rule ${skipped.rule} was not run — ${skipped.reason}.` - ); - } - - // Assembled against the snapshot base, whose layout mirrors the - // project's, so both configs resolve their rules inside the snapshot. - // The engines still run from the project root; only the config path - // is relative to it. - const assembled = await assembleEngineConfigs( - plan.snapshot.base, - selection === undefined ? {} : { ruleIds: selection.vale } - ); - const valeAssembly = - assembled.vale?.status === "ok" - ? { - ...assembled.vale, - path: fromProjectRoot(plan.snapshot, assembled.vale.path), - } - : assembled.vale; - const sgConfig = - assembled.sg === undefined - ? undefined - : fromProjectRoot(plan.snapshot, assembled.sg); - // `--rule` narrows WHAT runs; it does not widen what may run. A runtime - // rule named here is still subject to the plan. - const runtimeExecute = - selection === undefined - ? plan.execute - : plan.execute.filter((rule) => - selection.runtime.includes(rule.name) - ); - const dispatched = await runEngines({ - cwd, - paths: existingPaths, - // An `sg` selection that is empty means no ast-grep rule was named, - // so the engine has nothing to do and is skipped rather than being - // handed a filter that matches nothing — which would still spawn - // ast-grep, load every rule, and walk the project to report none. - astGrepConfigPath: - selection !== undefined && selection.sg.length === 0 - ? undefined - : sgConfig, - ...(selection === undefined ? {} : { astGrepRuleIds: selection.sg }), - vale: valeAssembly, - runtimeRules: runtimeExecute, + ...(selection === undefined ? {} : { selection }), runtimeTimeoutMs: parseTimeoutMs(args.timeout), - logs: run.logs, + preserve, + onPlan: (plan) => { + for (const notice of plan.notices) warnNotice(notice); + for (const failure of plan.failures) warn(`Error: ${failure}`); + for (const skipped of plan.skipped) { + warnNotice( + `runtime rule ${skipped.rule} was not run — ${skipped.reason}.` + ); + } + }, }); const results = dispatched.results; @@ -597,13 +547,13 @@ export const checkCommand = defineCommand({ ? {} : { entitlement: plan.entitlement }), ...(plan.integrity.length > 0 ? { integrity: plan.integrity } : {}), - ...(preserve ? { runDirectory: run.relativePath } : {}), + ...(preserve ? { runDirectory } : {}), }); console.log(JSON.stringify(output)); } else { console.log(formatText(results)); if (preserve) { - console.error(`Run directory kept: ${run.relativePath}`); + console.error(`Run directory kept: ${runDirectory}`); } } @@ -627,8 +577,6 @@ export const checkCommand = defineCommand({ console.error(message); } process.exitCode = 1; - } finally { - await run.close(); } } finally { // Concrete state event: a scan completed; counts only, no matched code. diff --git a/packages/cli/src/generated/report-v1.schema.json b/packages/cli/src/generated/report-v1.schema.json index 601afbdd..059ffff1 100644 --- a/packages/cli/src/generated/report-v1.schema.json +++ b/packages/cli/src/generated/report-v1.schema.json @@ -4,28 +4,33 @@ "properties": { "schemaVersion": { "type": "string", - "const": "1" + "const": "1", + "description": "Always `\"1\"`." }, "generatedAt": { "type": "string", "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z))$" + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z))$", + "description": "When the report was produced. `violations`, `files` and `findings` are counted at this moment." }, "producer": { "type": "object", "properties": { "kind": { "type": "string", - "const": "cli" + "const": "cli", + "description": "Always `cli`." }, "cliVersion": { "type": "string", "minLength": 1, - "maxLength": 64 + "maxLength": 64, + "description": "The Taskless CLI version that produced the report." } }, "required": ["kind", "cliVersion"], - "additionalProperties": false + "additionalProperties": false, + "description": "What produced the report." }, "scope": { "oneOf": [ @@ -34,26 +39,31 @@ "properties": { "provider": { "type": "string", - "const": "github" + "const": "github", + "description": "The repository is on GitHub." }, "owner": { "type": "string", "maxLength": 39, - "pattern": "^[\\dA-Za-z](?:[\\dA-Za-z]|-(?=[\\dA-Za-z]))*$" + "pattern": "^[\\dA-Za-z](?:[\\dA-Za-z]|-(?=[\\dA-Za-z]))*$", + "description": "The repository's GitHub owner." }, "repo": { "type": "string", "maxLength": 100, - "pattern": "^(?!\\.{1,2}$)[\\w.-]+$" + "pattern": "^(?!\\.{1,2}$)[\\w.-]+$", + "description": "The repository's name on GitHub." }, "url": { "type": "string", "maxLength": 200, - "pattern": "^https:\\/\\/github\\.com\\/[\\dA-Za-z-]+\\/[\\w.-]+$" + "pattern": "^https:\\/\\/github\\.com\\/[\\dA-Za-z-]+\\/[\\w.-]+$", + "description": "`https://github.com//`, for this owner and repo." }, "retention": { "type": "string", - "enum": ["public", "private"] + "enum": ["public", "private"], + "description": "Requested retention: `public` keeps the report 183 days and may name the repository in analytics; absent means `private`." }, "window": { "type": "object", @@ -61,16 +71,19 @@ "from": { "type": "string", "format": "date", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$", + "description": "First day of the window, inclusive." }, "to": { "type": "string", "format": "date", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$", + "description": "Last day of the window, inclusive." } }, "required": ["from", "to"], - "additionalProperties": false + "additionalProperties": false, + "description": "The period the PR history was mined over." } }, "required": ["provider", "owner", "repo", "url", "window"], @@ -81,12 +94,14 @@ "properties": { "provider": { "type": "string", - "const": "none" + "const": "none", + "description": "The repository has no GitHub remote." }, "label": { "type": "string", "minLength": 1, - "maxLength": 100 + "maxLength": 100, + "description": "A display name the user chose for the repository." }, "window": { "type": "object", @@ -94,22 +109,26 @@ "from": { "type": "string", "format": "date", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$", + "description": "First day of the window, inclusive." }, "to": { "type": "string", "format": "date", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$", + "description": "Last day of the window, inclusive." } }, "required": ["from", "to"], - "additionalProperties": false + "additionalProperties": false, + "description": "The period the report covers." } }, "required": ["provider", "window"], "additionalProperties": false } - ] + ], + "description": "The repository the report is about, and its window." }, "history": { "type": "object", @@ -117,17 +136,20 @@ "mergedPrs": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Merged PRs in `scope.window` that the agent mined." }, "reviewComments": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Total review comments on those PRs: every PR comment mined, whether or not it became a rule." }, "agentAssistedPrs": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Of `mergedPrs`, those an AI agent helped write." }, "patterns": { "type": "object", @@ -135,16 +157,19 @@ "ruleShaped": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Recurring review patterns a rule can enforce." }, "judgment": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Recurring review patterns that need human judgment, so no rule was proposed." } }, "required": ["ruleShaped", "judgment"], - "additionalProperties": false + "additionalProperties": false, + "description": "Recurring patterns the agent found in the review comments." } }, "required": [ @@ -153,7 +178,28 @@ "agentAssistedPrs", "patterns" ], - "additionalProperties": false + "additionalProperties": false, + "description": "The PR history the agent mined from GitHub over `scope.window`. Absent when it could not mine GitHub." + }, + "findings": { + "type": "object", + "properties": { + "violations": { + "type": "integer", + "minimum": 0, + "maximum": 1000000000, + "description": "Total `violations` across every written rule, at publish. Must equal the sum of the rules' `violations`." + }, + "files": { + "type": "integer", + "minimum": 0, + "maximum": 1000000000, + "description": "Distinct files with at least one violation of any rule, at publish. A file breaking three rules counts once, so it lies between the largest rule's `files` and the sum of all rules' `files`." + } + }, + "required": ["violations", "files"], + "additionalProperties": false, + "description": "What `taskless check` found in the repository's code at publish, across every written rule. Present exactly when every written rule carries `violations` and `files`." }, "rules": { "minItems": 1, @@ -165,57 +211,74 @@ "id": { "type": "string", "maxLength": 128, - "pattern": "^[\\dA-Za-z][\\w.-]*$" + "pattern": "^[\\dA-Za-z][\\w.-]*$", + "description": "The rule's id, unique within the report." }, "title": { "type": "string", "minLength": 1, - "maxLength": 200 + "maxLength": 200, + "description": "What the rule enforces, in one line, scrubbed by the CLI." }, "tier": { "type": "string", - "enum": ["cli", "cloud", "legacy"] + "enum": ["cli", "cloud", "legacy"], + "description": "Where the rule runs: `cli` on the Taskless CLI (sg, vale), `cloud` on Taskless Cloud (runtime), `legacy` in a quality tool the repository already ran, written by the agent during this work." }, "engine": { "type": "string", - "enum": ["sg", "vale", "runtime"] + "enum": ["sg", "vale", "runtime"], + "description": "The Taskless engine the rule runs on. Required for `cli` (sg or vale) and `cloud` (runtime) rules; absent on `legacy` rules." }, "tool": { "type": "string", "minLength": 1, "maxLength": 64, - "pattern": "^[\\w@./+-]+$" + "pattern": "^[\\w@./+-]+$", + "description": "The quality tool a `legacy` rule was written for (such as `eslint`), as an identifier. Only on `legacy` rules." }, "language": { "type": "string", "minLength": 1, - "maxLength": 40 + "maxLength": 40, + "description": "The language the rule checks, as display text (`TypeScript`)." + }, + "state": { + "type": "string", + "enum": ["written", "candidate"], + "description": "`written`: the rule exists in the repository and runs. `candidate`: a rule Taskless Cloud could write but has not; only in the `cloud` tier, and it carries no `violations` or `files`. A candidate is never counted as a rule." }, - "origin": { + "violations": { + "type": "integer", + "minimum": 0, + "maximum": 1000000000, + "description": "Times this rule matched the repository's code when the report was published: the rule's hits from `taskless check` (or, for a legacy rule, from its tool), at publish. Not history. Present on every written rule exactly when the report has `findings`." + }, + "files": { + "type": "integer", + "minimum": 0, + "maximum": 1000000000, + "description": "Files with at least one of this rule's `violations`, at publish. A count, never paths. At most `violations`, and 0 exactly when `violations` is 0." + }, + "surfaced": { "type": "object", "properties": { - "reviewComments": { + "prs": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Merged PRs in `scope.window` whose review raised this rule's pattern, from the mined PR history." }, - "prs": { + "reviewComments": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Review comments in those PRs that asked for this rule's pattern, within `scope.window`, from the mined PR history." } }, - "required": ["reviewComments", "prs"], - "additionalProperties": false - }, - "fixed": { - "type": "integer", - "minimum": 0, - "maximum": 1000000000 - }, - "fixedBasis": { - "type": "string", - "enum": ["measured", "estimated"] + "required": ["prs", "reviewComments"], + "additionalProperties": false, + "description": "How often reviewers raised this rule's pattern in `scope.window`: what the rule saves reviewers. Requires `history`." }, "complexity": { "anyOf": [ @@ -227,14 +290,16 @@ { "type": "null" } - ] + ], + "description": "A Cloud rule's cost multiplier, overriding `projection.inputs.cloudComplexity`; null for the default. Always null on `cli` and `legacy` rules." }, "tests": { "type": "object", "properties": { "status": { "type": "string", - "enum": ["pass", "fail", "refused", "not-run"] + "enum": ["pass", "fail", "refused", "not-run"], + "description": "The result of running the rule's tests at publish: `pass` (both examples behave), `fail`, `refused` (the engine would not run them), or `not-run`." }, "fail": { "anyOf": [ @@ -244,7 +309,8 @@ "code": { "type": "string", "minLength": 1, - "maxLength": 4000 + "maxLength": 4000, + "description": "The example's source code, scrubbed by the CLI." } }, "required": ["code"], @@ -253,7 +319,8 @@ { "type": "null" } - ] + ], + "description": "An example the rule flags, or null when there is none or it was withheld (see `withheld`)." }, "pass": { "anyOf": [ @@ -263,7 +330,8 @@ "code": { "type": "string", "minLength": 1, - "maxLength": 4000 + "maxLength": 4000, + "description": "The example's source code, scrubbed by the CLI." } }, "required": ["code"], @@ -272,7 +340,8 @@ { "type": "null" } - ] + ], + "description": "An example the rule accepts, or null when there is none or it was withheld (see `withheld`)." }, "withheld": { "maxItems": 2, @@ -280,11 +349,13 @@ "items": { "type": "string", "enum": ["fail", "pass"] - } + }, + "description": "Examples the CLI withheld because it could not scrub them; each listed example is null." } }, "required": ["status", "fail", "pass", "withheld"], - "additionalProperties": false + "additionalProperties": false, + "description": "The rule's tests and their two examples." } }, "required": [ @@ -292,100 +363,55 @@ "title", "tier", "language", - "fixed", - "fixedBasis", + "state", "complexity", "tests" ], "additionalProperties": false - } + }, + "description": "Every rule the report covers, written rules and candidates." }, "projection": { "type": "object", "properties": { - "perMonth": { - "type": "object", - "properties": { - "commentsPrevented": { - "type": "object", - "properties": { - "cli": { - "type": "integer", - "minimum": 0, - "maximum": 1000000000 - }, - "cloud": { - "type": "integer", - "minimum": 0, - "maximum": 1000000000 - }, - "legacy": { - "type": "integer", - "minimum": 0, - "maximum": 1000000000 - } - }, - "required": ["cli", "cloud"], - "additionalProperties": false - }, - "agentCatches": { - "type": "object", - "properties": { - "cli": { - "type": "integer", - "minimum": 0, - "maximum": 1000000000 - }, - "cloud": { - "type": "integer", - "minimum": 0, - "maximum": 1000000000 - }, - "legacy": { - "type": "integer", - "minimum": 0, - "maximum": 1000000000 - } - }, - "required": ["cli", "cloud"], - "additionalProperties": false - } - }, - "required": ["agentCatches"], - "additionalProperties": false - }, "inputs": { "type": "object", "properties": { "contextTokens": { "type": "integer", "exclusiveMinimum": 0, - "maximum": 10000000 + "maximum": 10000000, + "description": "Context tokens an agent call uses." }, "callsPerCatch": { "type": "number", "exclusiveMinimum": 0, - "maximum": 1000 + "maximum": 1000, + "description": "Agent calls a catch saves." }, "cloudComplexity": { "type": "number", "exclusiveMinimum": 0, - "maximum": 100 + "maximum": 100, + "description": "The default cost multiplier for a Cloud rule's catches." }, "pricePerMillionTokens": { "type": "number", "minimum": 0, - "maximum": 1000 + "maximum": 1000, + "description": "Dollars per million tokens." }, "minutesPerComment": { "type": "number", "minimum": 0, - "maximum": 600 + "maximum": 600, + "description": "Reviewer minutes a review comment costs." }, "reviewerHourlyUsd": { "type": "number", "minimum": 0, - "maximum": 10000 + "maximum": 10000, + "description": "A reviewer's hourly cost in dollars." } }, "required": [ @@ -396,11 +422,13 @@ "minutesPerComment", "reviewerHourlyUsd" ], - "additionalProperties": false + "additionalProperties": false, + "description": "The assumptions behind the value estimates; the page's sliders start here." } }, - "required": ["perMonth", "inputs"], - "additionalProperties": false + "required": ["inputs"], + "additionalProperties": false, + "description": "Assumptions for the estimated value a month. The service derives the monthly rates itself, from each rule's `surfaced` and the window." }, "redaction": { "type": "object", @@ -408,7 +436,8 @@ "policyVersion": { "type": "string", "minLength": 1, - "maxLength": 16 + "maxLength": 16, + "description": "The version of the CLI scrubbing policy applied." }, "counts": { "type": "object", @@ -416,42 +445,50 @@ "secret": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Secrets scrubbed." }, "email": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Email addresses scrubbed." }, "person": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "People's names scrubbed." }, "customer": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Customer names scrubbed." }, "host": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Hostnames scrubbed." }, "ip": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "IP addresses scrubbed." }, "phone": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Phone numbers scrubbed." }, "text": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Long strings scrubbed." } }, "required": [ @@ -464,16 +501,19 @@ "phone", "text" ], - "additionalProperties": false + "additionalProperties": false, + "description": "Values the CLI scrubbed from rule titles and examples, by kind." }, "withheldFixtures": { "type": "integer", "minimum": 0, - "maximum": 1000000000 + "maximum": 1000000000, + "description": "Examples withheld across all rules; equals the total of the rules' `tests.withheld`." } }, "required": ["policyVersion", "counts", "withheldFixtures"], - "additionalProperties": false + "additionalProperties": false, + "description": "What the CLI scrubbed before the report left the machine." } }, "required": [ @@ -517,6 +557,10 @@ "judgment": 4 } }, + "findings": { + "violations": 64, + "files": 23 + }, "rules": [ { "id": "use-api-client", @@ -524,12 +568,13 @@ "tier": "cli", "engine": "sg", "language": "TypeScript", - "origin": { - "reviewComments": 142, - "prs": 96 + "state": "written", + "violations": 64, + "files": 23, + "surfaced": { + "prs": 96, + "reviewComments": 142 }, - "fixed": 31, - "fixedBasis": "measured", "complexity": null, "tests": { "status": "pass", @@ -548,12 +593,11 @@ "tier": "cloud", "engine": "runtime", "language": "TypeScript", - "origin": { - "reviewComments": 88, - "prs": 63 + "state": "candidate", + "surfaced": { + "prs": 63, + "reviewComments": 88 }, - "fixed": 19, - "fixedBasis": "estimated", "complexity": 4, "tests": { "status": "not-run", @@ -566,16 +610,6 @@ } ], "projection": { - "perMonth": { - "commentsPrevented": { - "cli": 186, - "cloud": 105 - }, - "agentCatches": { - "cli": 116, - "cloud": 62 - } - }, "inputs": { "contextTokens": 150000, "callsPerCatch": 20, diff --git a/packages/cli/src/report/assemble.ts b/packages/cli/src/report/assemble.ts index 3d03c94d..f360770b 100644 --- a/packages/cli/src/report/assemble.ts +++ b/packages/cli/src/report/assemble.ts @@ -9,14 +9,13 @@ import { } from "./redact"; import { REPORT_FIXTURE_MAX_LENGTH } from "./schema"; import type { - PerTier, + Findings, RedactionKind, ReportRule, ReportScope, ReportTests, ReportV1, TestStatus, - Tier, } from "./types"; /** @@ -24,8 +23,8 @@ import type { * * The CLI's values win every conflict: the facts carry only what the agent * alone knows, and a fact about something the CLI measured (an engine, a test - * status) never reaches the payload. The projection is computed here and - * nowhere else. + * status) never reaches the payload. The monthly rates are not computed at + * all: tskl.es derives them from each rule's `surfaced` and the window. */ /** How many Taskless rules a report needs before it is worth publishing. */ @@ -59,8 +58,28 @@ function invalid(message: string): CLIError { return new CLIError(message, "REPORT_INVALID"); } +/** + * The draft's measured counts, when they still cover every written rule. + * + * The draft decided what it could measure among the rules it can see. A + * legacy rule is the one written rule it cannot see: it lives in another + * linter the CLI does not run, so its arrival in the facts withdraws every + * count rather than leaving a total that silently skips it. + */ +function measuredFindings( + draft: ReportDraft, + facts: ReportFacts +): Findings | undefined { + if (draft.findings === undefined || facts.legacy.length > 0) return undefined; + return draft.findings; +} + /** Pair each draft rule with its facts, and add the legacy and cloud rules. */ -function mergeRules(draft: ReportDraft, facts: ReportFacts): Candidate[] { +function mergeRules( + draft: ReportDraft, + facts: ReportFacts, + measured: boolean +): Candidate[] { const factsById = new Map(facts.rules.map((rule) => [rule.id, rule])); const localIds = new Set(draft.rules.map((rule) => rule.id)); @@ -77,7 +96,7 @@ function mergeRules(draft: ReportDraft, facts: ReportFacts): Candidate[] { .filter((id) => !factsById.has(id)); if (uncovered.length > 0) { throw invalid( - `facts are missing for these rules: ${uncovered.join(", ")}. Every rule \`taskless report\` lists needs a "rules" entry with its title, origin, and fix count.` + `facts are missing for these rules: ${uncovered.join(", ")}. Every rule \`taskless report\` lists needs a "rules" entry with its title and surfaced counts.` ); } @@ -91,9 +110,11 @@ function mergeRules(draft: ReportDraft, facts: ReportFacts): Candidate[] { tier: local.tier, engine: local.engine, language: local.language, - origin: ruleFacts.origin, - fixed: ruleFacts.fixed, - fixedBasis: ruleFacts.fixedBasis, + state: "written", + ...(measured && local.violations !== undefined + ? { violations: local.violations, files: local.files ?? 0 } + : {}), + surfaced: ruleFacts.surfaced, complexity: null, }, status: local.tests.status, @@ -110,9 +131,8 @@ function mergeRules(draft: ReportDraft, facts: ReportFacts): Candidate[] { tier: "legacy", tool: legacy.tool, language: legacy.language, - origin: legacy.origin, - fixed: legacy.fixed, - fixedBasis: legacy.fixedBasis, + state: "written", + surfaced: legacy.surfaced, complexity: null, }, // The CLI cannot run another linter's rule, so it does not claim it @@ -131,9 +151,10 @@ function mergeRules(draft: ReportDraft, facts: ReportFacts): Candidate[] { tier: "cloud", engine: "runtime", language: cloud.language, - origin: cloud.origin, - fixed: cloud.fixed, - fixedBasis: cloud.fixedBasis, + // Never counted: a candidate is a rule Cloud could write, not one + // that runs. + state: "candidate", + surfaced: cloud.surfaced, complexity: cloud.complexity, }, status: "not-run", @@ -249,32 +270,17 @@ function redactFixtures( }); } -/** Days in an inclusive window, at least one. */ -function windowDays(window: ReportFacts["window"]): number { - const from = Date.parse(`${window.from}T00:00:00Z`); - const to = Date.parse(`${window.to}T00:00:00Z`); - if (to < from) { +/** + * Refuse a window that ends before it starts. tskl.es divides by its length to + * derive the monthly rates, so a reversed one is caught here, by field name, + * rather than as a service validation error. + */ +function checkWindow(window: ReportFacts["window"]): void { + if (Date.parse(window.to) < Date.parse(window.from)) { throw invalid( `window.to (${window.to}) is before window.from (${window.from})` ); } - return Math.round((to - from) / 86_400_000) + 1; -} - -/** Sum a per-rule figure by tier, normalized to a 30-day month. */ -function perMonth( - rules: ReportRule[], - days: number, - pick: (rule: ReportRule) => number -): PerTier { - const totals: Record = { cli: 0, cloud: 0, legacy: 0 }; - for (const rule of rules) totals[rule.tier] += pick(rule); - const scale = (n: number) => Math.round((n * 30) / days); - return { - cli: scale(totals.cli), - cloud: scale(totals.cloud), - ...(totals.legacy > 0 ? { legacy: scale(totals.legacy) } : {}), - }; } function buildScope(draft: ReportDraft, facts: ReportFacts): ReportScope { @@ -309,7 +315,8 @@ export function assembleReport( facts: ReportFacts, now: Date = new Date() ): ReportV1 { - const candidates = mergeRules(draft, facts); + const findings = measuredFindings(draft, facts); + const candidates = mergeRules(draft, facts, findings !== undefined); enforceGate(candidates); const scope = buildScope(draft, facts); @@ -324,7 +331,7 @@ export function assembleReport( } const rules = redactFixtures(candidates, facts, tally); - const days = windowDays(facts.window); + checkWindow(facts.window); return { schemaVersion: "1", // Seconds precision: the schema's date-time pattern allows fractions, but a @@ -333,16 +340,9 @@ export function assembleReport( producer: draft.producer, scope, history: facts.history, + ...(findings === undefined ? {} : { findings }), rules, projection: { - perMonth: { - commentsPrevented: perMonth( - rules, - days, - (rule) => rule.origin?.reviewComments ?? 0 - ), - agentCatches: perMonth(rules, days, (rule) => rule.fixed), - }, inputs: { contextTokens: facts.estimates.contextTokens, callsPerCatch: facts.estimates.callsPerCatch, diff --git a/packages/cli/src/report/draft.ts b/packages/cli/src/report/draft.ts index 504bd2f6..f6b0f326 100644 --- a/packages/cli/src/report/draft.ts +++ b/packages/cli/src/report/draft.ts @@ -10,9 +10,11 @@ import { } from "../rules/engines"; import { isMissingDirectory } from "../rules/errno"; import { verifyOneRule } from "../rules/inspect"; +import { scanProject } from "../rules/scan-project"; import { resolveRepositoryContext } from "../util/git-remote"; import { CLI_VERSION } from "../version"; -import type { ProjectionInputs, TestStatus } from "./types"; +import { countFindings, type Measurement } from "./findings"; +import type { Findings, ProjectionInputs, TestStatus } from "./types"; /** * Everything about a report the CLI can establish on its own, without the @@ -36,6 +38,10 @@ export interface LocalRule { /** First fixture the rule accepts, or null when it has none. */ pass: string | null; }; + /** Matches in the code today, from the CLI's own scan; absent when unmeasured. */ + violations?: number; + /** Files holding those matches. */ + files?: number; } export interface DraftScope { @@ -52,6 +58,12 @@ export interface ReportDraft { scope: DraftScope; rules: LocalRule[]; inputs: Omit; + /** + * What the rules find in the code today, across every rule, from the CLI's + * own scan. Absent when any rule could not be measured; `unmeasured` says why. + */ + findings?: Findings; + unmeasured?: string; /** The facts the agent must still supply, by facts-file key. */ missing: string[]; } @@ -225,16 +237,63 @@ function missingFacts(rules: LocalRule[]): string[] { ]; } +/** + * Scan the project as `check --anonymous` would and count each rule's matches. + * + * Anonymous keeps the draft offline: no reconcile, so no request. A scan that + * throws leaves the report unmeasured rather than failing the draft, since + * every other fact in it is still worth printing. + */ +async function measureLocalRules( + cwd: string, + rules: LocalRule[] +): Promise { + const counted = rules.map(({ id, engine }) => ({ id, engine })); + // Nothing to scan, or a rule no anonymous scan runs: the count is decided + // without spending a scan on it. + if (rules.length === 0 || rules.some((rule) => rule.engine === "runtime")) { + return countFindings({ cwd, rules: counted, failures: [], results: [] }); + } + try { + const { plan, dispatched } = await scanProject(cwd, { + paths: [], + anonymous: true, + dangerouslyRunScripts: false, + }); + return countFindings({ + cwd, + rules: counted, + failures: [...plan.failures, ...dispatched.failures], + results: dispatched.results, + }); + } catch (error) { + return { + measured: false, + reason: `the scan failed: ${error instanceof Error ? error.message : String(error)}`, + }; + } +} + export async function buildDraft(cwd: string): Promise { - const [scope, rules] = await Promise.all([ + const [scope, collected] = await Promise.all([ resolveDraftScope(cwd), collectLocalRules(cwd), ]); + const measurement = await measureLocalRules(cwd, collected); + const rules = measurement.measured + ? collected.map((rule) => ({ + ...rule, + ...measurement.perRule.get(rule.id), + })) + : collected; return { producer: { kind: "cli", cliVersion: CLI_VERSION }, scope, rules, inputs: DEFAULT_PROJECTION_INPUTS, + ...(measurement.measured + ? { findings: measurement.findings } + : { unmeasured: measurement.reason }), missing: missingFacts(rules), }; } diff --git a/packages/cli/src/report/findings.ts b/packages/cli/src/report/findings.ts new file mode 100644 index 00000000..40dd8115 --- /dev/null +++ b/packages/cli/src/report/findings.ts @@ -0,0 +1,93 @@ +import { relative, resolve } from "node:path"; + +import type { CheckResult } from "../types/check"; +import type { Findings } from "./types"; + +/** + * What the rules find in the code today, counted by the CLI. + * + * All-or-nothing. tskl.es publishes `findings` only when every written rule + * carries a count, and a total that silently skipped a rule would be a number + * the reader cannot interpret. So either the CLI has a defensible count for + * every rule it was given, or it reports none and says why. + */ + +/** A rule the count is asked to cover. */ +export interface CountedRule { + id: string; + engine: "sg" | "vale" | "runtime"; +} + +export type Measurement = + | { measured: true; perRule: Map; findings: Findings } + | { measured: false; reason: string }; + +export function countFindings(options: { + cwd: string; + rules: CountedRule[]; + /** Plan and dispatch failures: any one means a rule may not have run. */ + failures: string[]; + results: CheckResult[]; +}): Measurement { + const { cwd, rules, failures, results } = options; + + // Reports are published logged out, so a runtime rule never runs here. + const runtime = rules.filter((rule) => rule.engine === "runtime"); + if (runtime.length > 0) { + return { + measured: false, + reason: `runtime rules do not run for a report: ${runtime.map((rule) => rule.id).join(", ")}`, + }; + } + if (failures.length > 0) { + return { + measured: false, + reason: `the scan did not complete: ${failures.join("; ")}`, + }; + } + + const files = new Map>( + rules.map((rule) => [rule.id, new Set()]) + ); + const violations = new Map(rules.map((rule) => [rule.id, 0])); + const allFiles = new Set(); + const unattributed = new Set(); + for (const result of results) { + const ruleFiles = files.get(result.ruleId); + if (ruleFiles === undefined) { + // ast-grep reports under the `id:` in the rule body. When that differs + // from the directory, counting on would publish a confident 0 for the + // rule while its real hits sat under a name nothing in the report has. + unattributed.add(result.ruleId); + continue; + } + // One spelling per file, so `./a.ts` and `a.ts` are counted once. + const file = relative(cwd, resolve(cwd, result.file)); + ruleFiles.add(file); + allFiles.add(file); + violations.set(result.ruleId, (violations.get(result.ruleId) ?? 0) + 1); + } + if (unattributed.size > 0) { + return { + measured: false, + reason: `results named rules the report does not have: ${[...unattributed].join(", ")}`, + }; + } + + const perRule = new Map( + rules.map((rule) => [ + rule.id, + { + violations: violations.get(rule.id) ?? 0, + files: files.get(rule.id)?.size ?? 0, + }, + ]) + ); + let total = 0; + for (const { violations: count } of perRule.values()) total += count; + return { + measured: true, + perRule, + findings: { violations: total, files: allFiles.size }, + }; +} diff --git a/packages/cli/src/report/summary.ts b/packages/cli/src/report/summary.ts index ba582876..cff01a6c 100644 --- a/packages/cli/src/report/summary.ts +++ b/packages/cli/src/report/summary.ts @@ -1,4 +1,4 @@ -import type { PerTier, ReportRule, ReportV1 } from "./types"; +import type { ReportRule, ReportV1 } from "./types"; /** * The brief rule summary printed after a publish. @@ -9,6 +9,11 @@ import type { PerTier, ReportRule, ReportV1 } from "./types"; * * The delete URL is deliberately absent. The summary names the record the CLI * can delete from, never the credential in it. + * + * No monthly rate either. tskl.es derives those, and a second copy of its + * formula here is a number that drifts from the page without anyone noticing. + * The summary states the two kinds of count it has, labelled apart: what the + * CLI found in the code today, and what review raised over the window. */ function tierLabel(rule: ReportRule): string { @@ -25,8 +30,20 @@ function tierLabel(rule: ReportRule): string { } } -function total(perTier: PerTier): number { - return perTier.cli + perTier.cloud + (perTier.legacy ?? 0); +function plural(count: number, one: string, many: string): string { + return `${String(count)} ${count === 1 ? one : many}`; +} + +/** What review raised over the window, summed across every rule. */ +function surfacedLine(report: ReportV1): string { + let prs = 0; + let comments = 0; + for (const rule of report.rules) { + prs += rule.surfaced?.prs ?? 0; + comments += rule.surfaced?.reviewComments ?? 0; + } + const { from, to } = report.scope.window; + return `Raised in review from ${from} to ${to}: ${plural(comments, "comment", "comments")} across ${plural(prs, "pull request", "pull requests")}.`; } export function renderSummary(options: { @@ -37,14 +54,19 @@ export function renderSummary(options: { invocation: string; }): string { const { report, url, expiresAt, record, invocation } = options; - const { perMonth } = report.projection; + const { findings } = report; const lines = [ "Taskless report published", "", `Rules (${String(report.rules.length)}):`, ...report.rules.map((rule) => ` - ${rule.title} (${tierLabel(rule)})`), "", - `Projected each month: ${String(total(perMonth.commentsPrevented))} review comments prevented, ${String(total(perMonth.agentCatches))} catches before review.`, + ...(findings === undefined + ? [] + : [ + `Found in the code today: ${plural(findings.violations, "problem", "problems")} in ${plural(findings.files, "file", "files")}.`, + ]), + surfacedLine(report), "", `View it: ${url}`, `Expires: ${expiresAt.slice(0, 10)}`, diff --git a/packages/cli/src/report/types.ts b/packages/cli/src/report/types.ts index 0a1d9fdc..b23690da 100644 --- a/packages/cli/src/report/types.ts +++ b/packages/cli/src/report/types.ts @@ -19,6 +19,21 @@ export interface ReportTests { withheld: ("fail" | "pass")[]; } +/** `written`: the rule exists and runs. `candidate`: a Cloud rule not yet written. */ +export type RuleState = "written" | "candidate"; + +/** What the rule's pattern cost reviewers over the window, from mined history. */ +export interface Surfaced { + prs: number; + reviewComments: number; +} + +/** What `check` found for one rule, or across all of them, at publish. */ +export interface Findings { + violations: number; + files: number; +} + export interface ReportRule { id: string; title: string; @@ -26,9 +41,11 @@ export interface ReportRule { engine?: "sg" | "vale" | "runtime"; tool?: string; language: string; - origin?: { reviewComments: number; prs: number }; - fixed: number; - fixedBasis: "measured" | "estimated"; + state: RuleState; + /** Present on every written rule exactly when the report carries `findings`. */ + violations?: number; + files?: number; + surfaced?: Surfaced; complexity: number | null; tests: ReportTests; } @@ -49,12 +66,6 @@ export type ReportScope = } | { provider: "none"; label: string; window: ReportWindow }; -export interface PerTier { - cli: number; - cloud: number; - legacy?: number; -} - export interface ProjectionInputs { contextTokens: number; callsPerCatch: number; @@ -85,11 +96,11 @@ export interface ReportV1 { agentAssistedPrs: number; patterns: { ruleShaped: number; judgment: number }; }; + /** Present exactly when every written rule carries `violations` and `files`. */ + findings?: Findings; rules: ReportRule[]; - projection: { - perMonth: { commentsPrevented: PerTier; agentCatches: PerTier }; - inputs: ProjectionInputs; - }; + /** Inputs only: tskl.es derives the monthly rates from `surfaced` and the window. */ + projection: { inputs: ProjectionInputs }; redaction: { policyVersion: string; counts: Record; diff --git a/packages/cli/src/rules/scan-project.ts b/packages/cli/src/rules/scan-project.ts new file mode 100644 index 00000000..66dca8e3 --- /dev/null +++ b/packages/cli/src/rules/scan-project.ts @@ -0,0 +1,106 @@ +import { assembleEngineConfigs } from "./assemble"; +import { runEngines, type DispatchResult } from "./dispatch"; +import { planCheck, type CheckPlan } from "./plan-check"; +import type { RuleSelection } from "./rule-filter"; +import { openRun } from "./run-directory"; +import { fromProjectRoot } from "./snapshot"; + +/** + * The scan `check` performs, callable from any command. + * + * `check` and `report` both run this, so a report's counts are the counts + * `check` would print for the same tree. Two scans that could drift apart are + * how a published number stops matching the one a user can reproduce. + */ + +export interface ScanOptions { + /** Target paths, already filtered to those that exist; empty is the whole project. */ + paths: string[]; + anonymous: boolean; + dangerouslyRunScripts: boolean; + /** `check --rule`: narrows what runs, never widens it. */ + selection?: RuleSelection; + runtimeTimeoutMs?: number; + /** Keep the run directory under `.taskless/.run/` instead of removing it. */ + preserve?: boolean; + /** + * Called once the plan is made and before any engine runs, so a caller can + * print the plan's notices ahead of the engines' own. + */ + onPlan?: (plan: CheckPlan) => void; +} + +export interface ScanResult { + plan: CheckPlan; + dispatched: DispatchResult; + /** The run directory, relative to the project; only kept when `preserve` was set. */ + runDirectory: string; +} + +export async function scanProject( + cwd: string, + options: ScanOptions +): Promise { + // This run's own directory under `.taskless/.run/`: the snapshot, the + // assembled configs, and the logs. Removed when the run ends, however + // it ends, unless `preserve` keeps it for debugging. + const run = await openRun(cwd, { preserve: options.preserve ?? false }); + try { + // Planned before dispatch, not during it: planning snapshots the rules + // tree and consults auth and reconcile state, which decides WHAT runs. + // Everything below reads the snapshot, never `.taskless/rules/`, so + // the bytes that run are the bytes that were judged. + const plan = await planCheck(cwd, run, { + anonymous: options.anonymous, + dangerouslyRunScripts: options.dangerouslyRunScripts, + }); + options.onPlan?.(plan); + + const { selection } = options; + // Assembled against the snapshot base, whose layout mirrors the + // project's, so both configs resolve their rules inside the snapshot. + // The engines still run from the project root; only the config path + // is relative to it. + const assembled = await assembleEngineConfigs( + plan.snapshot.base, + selection === undefined ? {} : { ruleIds: selection.vale } + ); + const valeAssembly = + assembled.vale?.status === "ok" + ? { + ...assembled.vale, + path: fromProjectRoot(plan.snapshot, assembled.vale.path), + } + : assembled.vale; + const sgConfig = + assembled.sg === undefined + ? undefined + : fromProjectRoot(plan.snapshot, assembled.sg); + // `--rule` narrows WHAT runs; it does not widen what may run. A runtime + // rule named here is still subject to the plan. + const runtimeExecute = + selection === undefined + ? plan.execute + : plan.execute.filter((rule) => selection.runtime.includes(rule.name)); + const dispatched = await runEngines({ + cwd, + paths: options.paths, + // An `sg` selection that is empty means no ast-grep rule was named, + // so the engine has nothing to do and is skipped rather than being + // handed a filter that matches nothing — which would still spawn + // ast-grep, load every rule, and walk the project to report none. + astGrepConfigPath: + selection !== undefined && selection.sg.length === 0 + ? undefined + : sgConfig, + ...(selection === undefined ? {} : { astGrepRuleIds: selection.sg }), + vale: valeAssembly, + runtimeRules: runtimeExecute, + runtimeTimeoutMs: options.runtimeTimeoutMs, + logs: run.logs, + }); + return { plan, dispatched, runDirectory: run.relativePath }; + } finally { + await run.close(); + } +} diff --git a/packages/cli/src/schemas/report-facts.ts b/packages/cli/src/schemas/report-facts.ts index 92e0968a..e25be981 100644 --- a/packages/cli/src/schemas/report-facts.ts +++ b/packages/cli/src/schemas/report-facts.ts @@ -5,10 +5,10 @@ import { z } from "zod"; * reads. * * NOT A REPORTV1. The CLI builds the report; the agent supplies only what its - * session taught it. Every object is strict, so a field the CLI owns — most - * pointedly `perMonth`, which the CLI computes — is refused rather than - * quietly merged, and the agent hears that it tried to set something it does - * not own. + * session taught it. Every object is strict, so a field someone else owns — + * the `violations` and `files` the CLI measures, the monthly rates tskl.es + * derives — is refused rather than quietly merged, and the agent hears that it + * tried to set something it does not own. * * Embedded into the `report` recipe as JSON Schema, so the recipe and this * file cannot describe two different shapes. @@ -26,14 +26,17 @@ const ruleId = z "The rule's id: its directory name, or one you choose for a rule outside .taskless/" ); -const origin = z +const surfaced = z .object({ + prs: count.describe( + "Pull or merge requests merged in the window with at least one review comment asking for what this rule enforces" + ), reviewComments: count.describe( - "Review comments in the window that asked for what this rule enforces" + "Review comments on those pull requests that ask for what this rule enforces, judged by reading each one. Bot and author comments count" ), - prs: count.describe("Pull or merge requests those comments were on"), }) - .strict(); + .strict() + .describe("How often review raised this rule's pattern over the window"); const outcome = { title: z @@ -43,13 +46,7 @@ const outcome = { .describe( "What the rule enforces, in a phrase a reader of the report understands" ), - origin, - fixed: count.describe( - "Times in the window the pattern was fixed after review: the catches this rule would have made" - ), - fixedBasis: z - .enum(["measured", "estimated"]) - .describe("`measured` when counted from history, `estimated` otherwise"), + surfaced, }; const fixture = z diff --git a/packages/cli/test/onboard.test.ts b/packages/cli/test/onboard.test.ts index bf2bc97a..e98952c4 100644 --- a/packages/cli/test/onboard.test.ts +++ b/packages/cli/test/onboard.test.ts @@ -316,9 +316,11 @@ describe("onboard recipe establishes the routing surface first", () => { }); }); -// Topic v6: onboarding asks one question, runs logged out, completes itself, +// Topic v7: onboarding asks one question, runs logged out, completes itself, // publishes a report, and ends on the CI offer. Each of these is a behavior an -// agent would otherwise improvise back toward the conversational v5 flow. +// agent would otherwise improvise back toward the conversational v5 flow. v7 +// adds the surfaced-counting method to the scan step, since the report recipe +// is only fetched after the mining it describes is over. describe("onboard recipe asks one question and finishes on its own", () => { let cwd: string; let stdout: string; @@ -342,8 +344,16 @@ describe("onboard recipe asks one question and finishes on its own", () => { : stdout.slice(start, start + title.length + next); } - it("is topic v6", () => { - expect(stdout).toMatch(/^# Topic: onboard {5}\(CLI v.+ \/ topic v6\)/m); + it("is topic v7", () => { + expect(stdout).toMatch(/^# Topic: onboard {5}\(CLI v.+ \/ topic v7\)/m); + }); + + it("gives the method for counting surfaced while mining, not after", () => { + const scan = step("Scan with high-signal filtering"); + expect(scan).toContain("merged in the"); + expect(scan).toContain("Bot and author comments count too"); + expect(scan).toContain("never by matching keywords"); + expect(scan).not.toMatch(/\bfixed\b/); }); it("asks the one question, set apart, through a question tool when there is one", () => { diff --git a/packages/cli/test/report-findings.test.ts b/packages/cli/test/report-findings.test.ts new file mode 100644 index 00000000..a8146d08 --- /dev/null +++ b/packages/cli/test/report-findings.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it } from "vitest"; + +import { countFindings, type CountedRule } from "../src/report/findings"; +import type { CheckResult } from "../src/types/check"; + +const cwd = "/repo"; + +function result(ruleId: string, file: string): CheckResult { + return { + source: "ast-grep", + ruleId, + severity: "warning", + message: `no ${ruleId}`, + file, + range: { start: { line: 0, column: 0 }, end: { line: 0, column: 1 } }, + matchedText: ruleId, + }; +} + +const sg = (id: string): CountedRule => ({ id, engine: "sg" }); +const vale = (id: string): CountedRule => ({ id, engine: "vale" }); + +describe("countFindings", () => { + it("counts every written rule, and the total as the sum", () => { + const measurement = countFindings({ + cwd, + rules: [sg("a"), sg("b"), sg("c"), vale("d")], + failures: [], + results: [ + ...Array.from({ length: 4 }, (_, n) => + result("a", `src/a${String(n)}.ts`) + ), + ...Array.from({ length: 75 }, () => result("b", "src/b.ts")), + ...Array.from({ length: 12 }, () => result("d", "README.md")), + ], + }); + expect(measurement.measured).toBe(true); + if (!measurement.measured) return; + expect(Object.fromEntries(measurement.perRule)).toEqual({ + a: { violations: 4, files: 4 }, + b: { violations: 75, files: 1 }, + c: { violations: 0, files: 0 }, + d: { violations: 12, files: 1 }, + }); + expect(measurement.findings).toEqual({ violations: 91, files: 6 }); + }); + + it("counts a file two rules break once in the total", () => { + const measurement = countFindings({ + cwd, + rules: [sg("a"), sg("b")], + failures: [], + results: [result("a", "src/x.ts"), result("b", "src/x.ts")], + }); + expect(measurement).toMatchObject({ + measured: true, + findings: { violations: 2, files: 1 }, + }); + }); + + it("treats two spellings of one path as one file", () => { + const measurement = countFindings({ + cwd, + rules: [sg("a")], + failures: [], + results: [ + result("a", "./src/x.ts"), + result("a", "src/x.ts"), + result("a", "/repo/src/x.ts"), + ], + }); + expect(measurement).toMatchObject({ + measured: true, + findings: { violations: 3, files: 1 }, + }); + }); + + it("measures nothing when a runtime rule is present", () => { + const measurement = countFindings({ + cwd, + rules: [sg("a"), { id: "tenant", engine: "runtime" }], + failures: [], + results: [], + }); + expect(measurement).toEqual({ + measured: false, + reason: "runtime rules do not run for a report: tenant", + }); + }); + + it("measures nothing when the scan reports a failure", () => { + const measurement = countFindings({ + cwd, + rules: [sg("a")], + failures: ["vale engine failed: boom"], + results: [result("a", "src/x.ts")], + }); + expect(measurement.measured).toBe(false); + }); + + it("measures nothing when a result names a rule the report lacks", () => { + // An sg rule whose body `id:` differs from its directory reports under + // the body's id. Counting on would publish a confident 0 for it. + const measurement = countFindings({ + cwd, + rules: [sg("a")], + failures: [], + results: [result("a", "src/x.ts"), result("renamed", "src/y.ts")], + }); + expect(measurement).toEqual({ + measured: false, + reason: "results named rules the report does not have: renamed", + }); + }); +}); diff --git a/packages/cli/test/report-telemetry.test.ts b/packages/cli/test/report-telemetry.test.ts index d397a2eb..10823736 100644 --- a/packages/cli/test/report-telemetry.test.ts +++ b/packages/cli/test/report-telemetry.test.ts @@ -108,9 +108,7 @@ describe("report telemetry", () => { rules: ["alpha", "beta", "gamma"].map((id) => ({ id, title: TITLE, - origin: { reviewComments: 1, prs: 1 }, - fixed: 1, - fixedBasis: "measured", + surfaced: { prs: 1, reviewComments: 1 }, })), estimates: { contextTokens: 1000, callsPerCatch: 2 }, }), diff --git a/packages/cli/test/report.test.ts b/packages/cli/test/report.test.ts index 92cf55b5..b4f968d1 100644 --- a/packages/cli/test/report.test.ts +++ b/packages/cli/test/report.test.ts @@ -47,8 +47,12 @@ function localRule(id: string, overrides: Partial = {}): LocalRule { }; } -function draftOf(rules: LocalRule[]): ReportDraft { +function draftOf( + rules: LocalRule[], + measured: Partial> = {} +): ReportDraft { return { + ...measured, producer: { kind: "cli", cliVersion: "0.12.0" }, scope: { provider: "github", @@ -77,9 +81,7 @@ function factsFor( rules: ids.map((id) => ({ id, title: `Rule ${id}`, - origin: { reviewComments: 9, prs: 6 }, - fixed: 10, - fixedBasis: "measured", + surfaced: { prs: 6, reviewComments: 9 }, })), estimates: { contextTokens: 150_000, callsPerCatch: 20 }, ...overrides, @@ -144,28 +146,129 @@ describe("assembleReport", () => { }); }); - it("normalizes per-tier counts to a 30-day month over the window", () => { - // 2026-07-14 through 2026-10-11 inclusive is 90 days; three rules fixed - // 10 times each is 30, so 10 a month. + it("sends projection inputs and no rates", () => { + // tskl.es derives the monthly rates itself and refuses a payload that + // carries them. const report = assembleReport( draftOf(three.map((id) => localRule(id))), factsFor(three) ); - expect(report.projection.perMonth.agentCatches.cli).toBe(10); + expect(Object.keys(report.projection)).toEqual(["inputs"]); expect(report.projection.inputs).toMatchObject({ contextTokens: 150_000, callsPerCatch: 20, }); }); + it("marks local rules written and carries their surfaced counts", () => { + const report = assembleReport( + draftOf(three.map((id) => localRule(id))), + factsFor(three) + ); + expect(report.rules[0]).toMatchObject({ + state: "written", + surfaced: { prs: 6, reviewComments: 9 }, + }); + }); + + it("carries the draft's measured findings on every written rule", () => { + const report = assembleReport( + draftOf( + [ + localRule("a", { violations: 4, files: 2 }), + localRule("b", { violations: 0, files: 0 }), + localRule("c", { violations: 7, files: 3 }), + ], + { findings: { violations: 11, files: 4 } } + ), + factsFor(three) + ); + expect(reportSchemaIssues(report)).toEqual([]); + expect(report.findings).toEqual({ violations: 11, files: 4 }); + expect(report.rules.map((rule) => [rule.violations, rule.files])).toEqual([ + [4, 2], + [0, 0], + [7, 3], + ]); + }); + + it("publishes no counts when the draft could not measure", () => { + const report = assembleReport( + draftOf( + three.map((id) => localRule(id)), + { + unmeasured: "the scan did not complete", + } + ), + factsFor(three) + ); + expect(reportSchemaIssues(report)).toEqual([]); + expect(report.findings).toBeUndefined(); + expect(report.rules.some((rule) => "violations" in rule)).toBe(false); + }); + + it("withdraws every count when the facts add a legacy rule", () => { + // The CLI cannot run another linter's rule, so a total over the rules it + // can see would silently skip one. + const report = assembleReport( + draftOf( + three.map((id) => localRule(id, { violations: 1, files: 1 })), + { findings: { violations: 3, files: 1 } } + ), + factsFor(three, { + legacy: [ + { + id: "eslint-rule", + title: "No default exports", + surfaced: { prs: 1, reviewComments: 1 }, + tool: "eslint", + language: "TypeScript", + }, + ], + }) + ); + expect(reportSchemaIssues(report)).toEqual([]); + expect(report.findings).toBeUndefined(); + expect(report.rules.some((rule) => "violations" in rule)).toBe(false); + expect(report.rules.find((rule) => rule.id === "eslint-rule")?.state).toBe( + "written" + ); + }); + + it.each([ + ["fixed", { fixed: 3 }], + ["fixedBasis", { fixedBasis: "measured" }], + ["origin", { origin: { reviewComments: 1, prs: 1 } }], + ["violations", { violations: 3 }], + ["files", { files: 1 }], + ["state", { state: "written" }], + ])("refuses a rule's %s in the facts", (_field, extra) => { + const facts = { + ...factsFor(three), + rules: three.map((id) => ({ + id, + title: `Rule ${id}`, + surfaced: { prs: 1, reviewComments: 1 }, + ...extra, + })), + }; + expect(reportFactsSchema.safeParse(facts).success).toBe(false); + }); + + it("refuses findings in the facts", () => { + const facts = { + ...factsFor(three), + findings: { violations: 342, files: 118 }, + }; + expect(reportFactsSchema.safeParse(facts).success).toBe(false); + }); + it("refuses with too few local rules, cloud candidates notwithstanding", () => { const facts = factsFor(["a", "b"], { cloud: ["x", "y", "z", "w"].map((id) => ({ id, title: `Cloud ${id}`, - origin: { reviewComments: 1, prs: 1 }, - fixed: 1, - fixedBasis: "estimated", + surfaced: { prs: 1, reviewComments: 1 }, language: "TypeScript", complexity: 3, })), @@ -185,9 +288,7 @@ describe("assembleReport", () => { legacy: ["l1", "l2"].map((id) => ({ id, title: `Legacy ${id}`, - origin: { reviewComments: 1, prs: 1 }, - fixed: 1, - fixedBasis: "measured", + surfaced: { prs: 1, reviewComments: 1 }, tool: "eslint", language: "TypeScript", })), @@ -227,9 +328,7 @@ describe("assembleReport", () => { { id: "tenant-check", title: "Routes call requireTenant()", - origin: { reviewComments: 4, prs: 3 }, - fixed: 2, - fixedBasis: "estimated", + surfaced: { prs: 3, reviewComments: 4 }, language: "TypeScript", complexity: 4, fail: "export const GET = route(() => db.all())", @@ -245,6 +344,7 @@ describe("assembleReport", () => { ).toMatchObject({ tier: "cloud", engine: "runtime", + state: "candidate", tests: { status: "not-run" }, }); }); @@ -298,18 +398,14 @@ describe("assembleReport", () => { const legacy = { id: "l1", title: "Legacy l1", - origin: { reviewComments: 1, prs: 1 }, - fixed: 1, - fixedBasis: "measured" as const, + surfaced: { prs: 1, reviewComments: 1 }, tool: "eslint", language: "TypeScript", }; const cloud = { id: "c1", title: "Cloud c1", - origin: { reviewComments: 1, prs: 1 }, - fixed: 1, - fixedBasis: "estimated" as const, + surfaced: { prs: 1, reviewComments: 1 }, language: "TypeScript", complexity: 3, }; @@ -383,6 +479,16 @@ describe("scanText", () => { }); }); +function summaryOf(draft: ReportDraft): string { + return renderSummary({ + report: assembleReport(draft, factsFor(["a", "b", "c"])), + url: "https://tskl.es/r/abc", + expiresAt: "2026-11-12T00:00:00Z", + record: "2026-10-13_142200.json", + invocation: "npx @taskless/cli", + }); +} + describe("renderSummary", () => { it("names each rule and the link, and never the delete URL", () => { const report = assembleReport( @@ -401,6 +507,32 @@ describe("renderSummary", () => { expect(summary).toContain("can be deleted after review"); expect(summary).not.toContain("delete/"); }); + + it("leads with what the rules found today, apart from review history", () => { + const summary = summaryOf( + draftOf( + ["a", "b", "c"].map((id) => + localRule(id, { violations: 114, files: 40 }) + ), + { findings: { violations: 342, files: 118 } } + ) + ); + expect(summary).toContain( + "Found in the code today: 342 problems in 118 files." + ); + expect(summary).toContain( + "Raised in review from 2026-07-14 to 2026-10-11: 27 comments across 18 pull requests." + ); + expect(summary).not.toMatch(/month/i); + }); + + it("states no problem count it does not have", () => { + const summary = summaryOf( + draftOf(["a", "b", "c"].map((id) => localRule(id))) + ); + expect(summary).not.toContain("Found in the code today"); + expect(summary).toContain("Raised in review"); + }); }); interface FakeResponse { @@ -597,6 +729,10 @@ describe("taskless report", () => { cwd = await mkdtemp(join(tmpdir(), "tskl-report-")); await run(["init"]); for (const id of ["alpha", "beta", "gamma"]) await sgRule(id); + // Three matches across two files: alpha twice in one, beta once in each + // file, gamma nowhere. + await writeFile(join(cwd, "one.ts"), "alpha(1);\nalpha(2);\nbeta(3);\n"); + await writeFile(join(cwd, "two.ts"), "beta(4);\n"); }); afterEach(async () => { @@ -620,6 +756,14 @@ describe("taskless report", () => { true ); expect(draft.scope.provider).toBe("none"); + expect(draft.findings).toEqual({ violations: 4, files: 2 }); + expect( + draft.rules.map((rule) => [rule.id, rule.violations, rule.files]) + ).toEqual([ + ["alpha", 2, 1], + ["beta", 2, 2], + ["gamma", 0, 0], + ]); expect(draft.missing).toContain("history"); expect(draft.missing).toContain("rules[alpha]"); }); @@ -656,6 +800,10 @@ describe("taskless report", () => { const sent = JSON.parse(fake.requests[0]!.body) as Record; expect(reportSchemaIssues(sent)).toEqual([]); + expect(sent.findings).toEqual({ violations: 4, files: 2 }); + expect(published.stdout).toContain( + "Found in the code today: 4 problems in 2 files." + ); const records = await recordsIn(cwd); expect(records).toHaveLength(1); diff --git a/packages/cli/test/scan-project.test.ts b/packages/cli/test/scan-project.test.ts new file mode 100644 index 00000000..f3d6552a --- /dev/null +++ b/packages/cli/test/scan-project.test.ts @@ -0,0 +1,82 @@ +import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ensureTasklessDirectory } from "../src/filesystem/directory"; +import { scanProject } from "../src/rules/scan-project"; +import { findValeBinary } from "../src/rules/vale/binary"; + +/** + * `report` runs this scan to measure what the rules find, and a report's draft + * must stay offline. Anonymous is what keeps it there: no reconcile, so no + * request, and every runtime rule skipped rather than run unverified. + */ + +const fixturesDirectory = resolve( + import.meta.dirname, + "fixtures/mixed-engines-project" +); + +const RUNTIME_CAPTURE = [ + "id: logs", + "language: TypeScript", + "rule:", + " pattern: console.log($$$)", + "metadata:", + " taskless:", + " version: 1", + " kind: runtime", + " name: logs", + " check: check.ts", + " match: anchor", + "", +].join("\n"); + +const RUNTIME_CHECK = `export default async function (root, matches) { + return matches.map((m) => ({ file: m.file, line: m.line, message: "runtime", severity: "warning" })); +} +`; + +/** Vale ships per-platform; an unsupported host has none. */ +const withVale = findValeBinary().path === undefined ? describe.skip : describe; + +withVale("scanProject, anonymous", () => { + let project: string; + + beforeEach(async () => { + project = await mkdtemp(join(tmpdir(), "taskless-scan-")); + await cp(fixturesDirectory, project, { recursive: true }); + const runtime = join(project, ".taskless", "rules", "runtime", "demo"); + await mkdir(join(runtime, "captures"), { recursive: true }); + await writeFile(join(runtime, "captures", "logs.yml"), RUNTIME_CAPTURE); + await writeFile(join(runtime, "check.ts"), RUNTIME_CHECK); + await ensureTasklessDirectory(project, { onNotice: () => {} }); + vi.stubGlobal( + "fetch", + vi.fn(() => { + throw new Error("the anonymous scan made a network request"); + }) + ); + }); + + afterEach(async () => { + vi.unstubAllGlobals(); + await rm(project, { recursive: true, force: true }); + }); + + it("runs sg and vale, skips runtime, and makes no request", async () => { + const { plan, dispatched } = await scanProject(project, { + paths: [], + anonymous: true, + dangerouslyRunScripts: false, + }); + + expect(fetch).not.toHaveBeenCalled(); + expect(plan.execute).toEqual([]); + expect(plan.skipped.map((skipped) => skipped.rule)).toContain("demo"); + const sources = new Set(dispatched.results.map((result) => result.source)); + expect(sources).toEqual(new Set(["ast-grep", "vale"])); + }); +});