From 8de2b4d4478df1e48d52d3594de9e7ea0a9815a5 Mon Sep 17 00:00:00 2001 From: chumanic Date: Wed, 4 Nov 2026 16:48:37 +0800 Subject: [PATCH] feat: export bounded JUnit verification and inventory reports --- README.md | 6 +- docs/ci.md | 28 ++++++ docs/reviews/2026-11-02-project-review.md | 107 ---------------------- src/cli/commands/scan.ts | 2 + src/cli/index.ts | 11 ++- src/reporters/junit.ts | 77 ++++++++++++++++ src/workbench/page.ts | 2 +- tests/cli/junit.test.ts | 22 +++++ tests/docs/documentation.test.ts | 2 +- tests/reporters/junit.test.ts | 33 +++++++ 10 files changed, 176 insertions(+), 114 deletions(-) delete mode 100644 docs/reviews/2026-11-02-project-review.md create mode 100644 src/reporters/junit.ts create mode 100644 tests/cli/junit.test.ts create mode 100644 tests/reporters/junit.test.ts diff --git a/README.md b/README.md index 6541d71..3d0f9f6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # SkillSync -**Verify Agent Skills before you trust them.** SkillSync checks a local Skill's provenance, compatibility, and changes without executing it. +Inspect local Skill folders and declared artifact deliveries. Review provenance, compatibility and changes without executing the Skill. > **Alpha · GitHub release v0.3.0 · Node.js 20+** > SkillSync performs offline checks of local Skill content. It does not execute Skill scripts, does not read credentials, and does not enable live provider, remote-worker, or runtime capabilities. @@ -11,6 +11,10 @@ GitHub v0.3.0 adds a declared cross-file checklist to HTML, text and JSON report ## Local graphical workbench +Current source also exports `verify` and `scan` results as JUnit XML for CI test +views. Build this checkout and use `--format junit`; the existing policy exit +code stays unchanged. [Report mapping and limits](docs/ci.md#junit-test-reports-current-source). + Run `skillsync gui --state-dir /absolute/private/report-directory`, then open `http://127.0.0.1:14986`. Select a Skill directory, or select a contract and its artifact directory. The macOS folder/file picker also accepts an explicit path. diff --git a/docs/ci.md b/docs/ci.md index 4124484..9415b6b 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -30,8 +30,36 @@ The default regression also runs the live-runtime preparation integration and documentation tests. Its package step is `npm pack --dry-run`; it does not publish or resolve any deployment-owned reference. +## JUnit test reports (current source) + +Build the current source checkout, then export a test report: + +```bash +node dist/cli/index.js verify --path ./skills --target codex --format junit > skillsync.junit.xml +node dist/cli/index.js scan --path ./skills --format junit > inventory.junit.xml +``` + +Upload the XML through your CI's existing test-report step even when verification +returns nonzero; retain that original exit code. This addition is not in the +unchanged npm 0.1.0 or GitHub v0.3.0 archive, and enables no upload or workflow by +itself. Other commands keep their existing format choices. + +Each finding is one uniquely named testcase. `fail` becomes a failure; +`warn` and `unknown` become skipped checks, not successful verification. Verify +adds a separate policy testcase: a nonzero policy result fails it, and invalid +input (exit2) is an error. The finding and policy outcomes remain separate; a +policy allowing a failed finding does not relabel that individual finding as a +pass. An empty scan records only inventory completion, not Skill acceptance. + +Reports omit raw evidence, prompts and source bodies. Existing path-redaction +policy is honored. XML characters and invalid code points are escaped or +replaced; no DTD or external resources are emitted. Text fields are capped at +2,048 UTF-16 units with an ellipsis. More than 10,000 total testcases or 2 MiB of +UTF-8 XML fails explicitly before any partial report is returned. + ## Runtime canary contracts + `.github/workflows/skillsync-runtime-canary.yml` is manual-only and requires a named GitHub Environment so the canary evidence has an explicit owner. Its default path runs only offline egress, credential-reference, activation-gate, diff --git a/docs/reviews/2026-11-02-project-review.md b/docs/reviews/2026-11-02-project-review.md deleted file mode 100644 index e9bac85..0000000 --- a/docs/reviews/2026-11-02-project-review.md +++ /dev/null @@ -1,107 +0,0 @@ -# SkillSync project status and optimization review - -Status: reviewed source and follow-up plan. Date: 2026-11-02, Asia/Shanghai. - -Reviewed baseline: `24bde228cb76441342addaabbc5cd8a377f14bb3`. -The implementation references below apply to that baseline. This change adds a -review document; the proposed implementation changes remain follow-up work. - -## Current stage - -SkillSync provides an offline Alpha workflow for declared provenance, -compatibility, behavior checks, and physical artifact review. It does not -authenticate a remote publisher, certify execution isolation, or establish -business acceptance. - -## Implemented capabilities - -- The CLI covers inventory, verify, compat, diff, fix, report, lock, and behavior - checks. Stable content hashing and path constraints are implemented in - [the CLI entry point](../../src/cli/index.ts) and - [digest handling](../../src/domain/digest.ts#L18). -- Artifact contracts, manifests, receipts, physical files, reports, and batch - review packages are connected. The implementation includes digest binding, - references, keyed reconciliation, and capacity limits. See - [artifact commands](../../src/cli/commands/artifacts.ts#L81) and - [batch reports](../../src/domain/batch-report.ts#L57). -- The loopback Workbench supports package switching, explicit review, - cancellation, history, and batch review. It uses native HTML and JavaScript; - see [the server](../../src/workbench/server.ts#L31) and - [the page](../../src/workbench/page.ts#L15). - -## Completion boundaries - -- The Git provenance branch checks that a URL and a plausible resolved commit - are present. It emits `provenance.verified` without checking that the named - commit contains the current files. See - [the provenance scanner](../../src/scanners/provenance.ts#L92). -- Artifact review explicitly retains `provenance: not-authenticated`. Signed - issuers, live providers, remote workers, and production isolation require - separate design and acceptance. See - [the security boundary](../../docs/security-boundary.md#L58) and - [artifact delivery](../../docs/artifact-delivery.md#L18). -- `reference_exists` currently supports a unique string key in CSV. JSON - relationships and composite references are outside that contract. See - [the artifact contract](../../src/domain/artifact-contract.ts#L115). - -## Prioritized follow-up - -P1 identifies a correctness or delivery boundary to resolve first. P2 covers -reliability and maintenance. P3 covers measurements or later extensions. -These priorities are not vulnerability ratings. - -### P1 Align provenance labels with the evidence actually checked - -Evidence: direct source inspection; no exploit or runtime reproduction was run. -A URL and syntactically valid commit can produce a `verified` finding without -proving a remote-content identity. - -Acceptance: use a declared-source/local-comparison status, or introduce an -explicitly controlled Git-object or signature verifier. Cover a present source -declaration whose content does not match the named commit. Keep offline and -authenticated evidence distinct. - -References: [scanner](../../src/scanners/provenance.ts#L92), -[documented boundary](../../docs/security-boundary.md#L14). - -### P1 Enforce consistent artifact report states - -Evidence: a schema constraint is absent in the reviewed source; a constructed -payload has not been executed in this review. `reportSchema` does not explicitly -reject `status: passed` together with `receipt_binding: failed`. - -Acceptance: enforce the state implication, cover passed/failed reports against -all receipt states, and keep downstream comparisons closed to incomplete -evidence. See [report parsing](../../src/domain/artifact-report.ts#L58). - -### P2 Bind batch declarations to the canonical contract - -Checking a contract digest alone does not establish that the report's -declarations match that contract's field mappings and limits. - -Acceptance: require canonical declaration equality for successful packages. -Cover absent or altered declarations in both delivery modes. See -[batch report checks](../../src/domain/batch-report.ts#L60) and -[batch commands](../../src/cli/commands/batch.ts#L98). - -### P3 Measure repeated scans before adding bounded reuse - -Evidence: source-based performance hypothesis; no timing or RSS baseline was -collected. Workbench checks and batch retries can repeat scanning and hashing. - -Acceptance: measure representative and near-limit inputs first. Reuse one -operation's immutable snapshot or a bounded cache with strict identity-based -invalidation; preserve change detection and fail-closed behavior. See -[Workbench scans](../../src/workbench/server.ts#L177) and -[batch retries](../../src/cli/commands/batch.ts#L109). - -## Verification record - -- The reviewed baseline had successful `verify` and `regression` GitHub checks: - [baseline workflow](https://github.com/Chumaniac/skillsync/actions/runs/41275822538/job/124757678044). -- The initial review commit triggered the repository's existing English-only - Markdown check. This revision corrects the document language without changing - the check, its assertions, or implementation code. -- The implementation concerns above are source findings, not completed fixes. - No live provider, remote runner, Docker canary, or new identity trust root was - enabled. diff --git a/src/cli/commands/scan.ts b/src/cli/commands/scan.ts index 38d3065..de2557a 100644 --- a/src/cli/commands/scan.ts +++ b/src/cli/commands/scan.ts @@ -8,6 +8,7 @@ import { } from "../../domain/inventory.js"; import type { Finding } from "../../domain/result.js"; import { formatOutput, parseOutputFormat, type OutputFormat } from "../output.js"; +import { renderJUnit } from "../../reporters/junit.js"; export type ScanOptions = InventoryOptions & { paths: string[]; @@ -63,6 +64,7 @@ export async function runScan(options: ScanOptions): Promise { } export function renderScan(report: ScanReport, format: string | undefined): string { + if (format === "junit") return renderJUnit(report, "scan"); const outputFormat: OutputFormat = parseOutputFormat(format); return formatOutput(report, outputFormat); } diff --git a/src/cli/index.ts b/src/cli/index.ts index 79aff30..3c9036b 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -62,6 +62,7 @@ import { renderSarif } from "../reporters/sarif.js"; import { renderText } from "../reporters/text.js"; import { renderVerificationHtml } from "../reporters/verification-html.js"; import { parseOutputFormat } from "./output.js"; +import { renderJUnit } from "../reporters/junit.js"; import { VERSION } from "../version.js"; @@ -265,7 +266,7 @@ export function createCli(io: CliIO = defaultCliIO): Command { .description("Inspect skill files without executing their instructions.") .option("--path ", "Explicit Skill directory or directory containing Skills") .option("--follow-symlinks", "Follow only symlinks that remain inside the Skill root") - .option("--format ", "Output format: text, json, or sarif", "text") + .option("--format ", "Output format: text, json, sarif, or junit", "text") .action(async (options: ScanCliOptions) => { const report = await runScan({ paths: options.path ?? [], @@ -303,7 +304,7 @@ export function createCli(io: CliIO = defaultCliIO): Command { .option("--policy ", "YAML or JSON policy file") .option("--profile-dir ", "Additional directory containing external profile YAML") .option("--follow-symlinks", "Follow only symlinks that remain inside the Skill root") - .option("--format ", "Output format: text, json, sarif, or offline html", "text") + .option("--format ", "Output format: text, json, sarif, junit, or offline html", "text") .action(async (options: VerifyCliOptions) => { const report = await runVerification({ paths: options.path ?? [], @@ -312,8 +313,10 @@ export function createCli(io: CliIO = defaultCliIO): Command { profileDir: options.profileDir, followSymlinks: options.followSymlinks, }); - const format = options.format === "html" ? "html" : parseOutputFormat(options.format); - const output = format === "html" + const format = options.format === "html" || options.format === "junit" ? options.format : parseOutputFormat(options.format); + const output = format === "junit" + ? renderJUnit(report, "verify") + : format === "html" ? renderVerificationHtml(report) : format === "json" ? renderJson(report) diff --git a/src/reporters/junit.ts b/src/reporters/junit.ts new file mode 100644 index 0000000..d891579 --- /dev/null +++ b/src/reporters/junit.ts @@ -0,0 +1,77 @@ +import type { Finding } from "../domain/result.js"; +import { redactLocalPaths } from "./local-paths.js"; + +type JUnitReport = { + findings: Finding[]; + exitCode?: number; + reporting?: { include_local_paths?: boolean }; +}; + +const MAX_CASES = 10_000; +const MAX_BYTES = 2 * 1024 * 1024; + +function xml(value: string): string { + let safe = ""; + for (const character of value.slice(0, 2048)) { + const point = character.codePointAt(0)!; + safe += point === 9 || point === 10 || point === 13 || + point >= 0x20 && point <= 0xd7ff || point >= 0xe000 && point <= 0xfffd || + point >= 0x10000 && point <= 0x10ffff ? character : "�"; + } + if (value.length > 2048) safe += "…"; + const entities: Record = { "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }; + return safe.replace(/[&<>"']/g, character => entities[character]); +} + +/** Static checks only. Raw evidence and source contents never enter this format. */ +export function renderJUnit(report: JUnitReport, command: "verify" | "scan"): string { + const gate = report.exitCode !== undefined || report.findings.length === 0; + const total = report.findings.length + Number(gate); + if (total > MAX_CASES) throw new Error("JUnit report capacity exceeded"); + if (report.exitCode !== undefined && (!Number.isInteger(report.exitCode) || report.exitCode < 0 || report.exitCode > 4)) { + throw new Error("Invalid JUnit policy exit code"); + } + let failures = 0, skipped = 0; + for (const finding of report.findings) { + switch (finding.status) { + case "fail": failures++; break; + case "warn": case "unknown": skipped++; break; + case "pass": break; + default: throw new Error("Invalid JUnit finding status"); + } + } + const policyFailed = report.exitCode !== undefined && report.exitCode !== 0; + const errors = Number(report.exitCode === 2); + if (policyFailed && !errors) failures++; + const parts: string[] = []; + let bytes = 0; + const append = (part: string) => { + bytes += Buffer.byteLength(part, "utf8"); + if (bytes > MAX_BYTES) throw new Error("JUnit report capacity exceeded"); + parts.push(part); + }; + append(`\n\n`); + append('\n'); + for (const [index, finding] of report.findings.entries()) { + const safe = redactLocalPaths({ code: finding.code, skill: finding.skill, target: finding.target, + message: finding.message, remediation: finding.remediation }, { includeLocalPaths: report.reporting?.include_local_paths === true }); + const name = `${safe.code} :: ${safe.skill}${safe.target ? ` (${safe.target})` : ""} #${index + 1}`; + append(``); + if (finding.status === "fail") { + append(`${xml(safe.remediation ?? safe.message)}`); + } else if (finding.status === "warn" || finding.status === "unknown") { + append(``); + } + append("\n"); + } + if (gate) { + append(``); + if (policyFailed) { + const kind = errors ? "error" : "failure"; + append(`<${kind} type="policy-exit" message="Verification exit code ${report.exitCode}"/>`); + } + append("\n"); + } + append("\n"); + return parts.join(""); +} diff --git a/src/workbench/page.ts b/src/workbench/page.ts index 68596c1..3058044 100644 --- a/src/workbench/page.ts +++ b/src/workbench/page.ts @@ -1,7 +1,7 @@ /** Static local UI. All evidence is inserted as text or in a scriptless report frame. */ export function workbenchPage(nonce: string): string { return `SkillSync 核验台
-

SkillSync

本机核验台

选择要检查的目录或交付，核对证据后再决定下一步。

+

SkillSync

本机核验台

打开 Skill 目录或交付包，查看核验结果、历史记录与修复计划。

diff --git a/tests/cli/junit.test.ts b/tests/cli/junit.test.ts new file mode 100644 index 0000000..5ba7094 --- /dev/null +++ b/tests/cli/junit.test.ts @@ -0,0 +1,22 @@ +import { access, mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { runCli } from "../../src/cli/index.js"; + +it("verify emits JUnit while keeping its original policy exit and never executing source scripts", async () => { + const root = await mkdtemp(join(tmpdir(), "skillsync-junit-")), marker = join(root, "executed.marker"); + await writeFile(join(root, "SKILL.md"), "---\nname: review\ndescription: Review\n---\n[missing](absent.md)\n"); + await writeFile(join(root, "check.sh"), `touch ${marker}\n`); + const original = await runCli(["verify", "--path", root, "--target", "codex", "--format", "json"]); + const junit = await runCli(["verify", "--path", root, "--target", "codex", "--format", "junit"]); + expect(junit.exitCode).toBe(original.exitCode); + expect(junit.stdout).toMatch(/^<\?xml version="1.0" encoding="UTF-8"\?>/); + expect(junit.stdout).toContain("structure.missing-reference"); + expect(junit.stdout).not.toContain(root); + await expect(access(marker)).rejects.toThrow(); + const scan = await runCli(["scan", "--path", root, "--format", "junit"]); + expect(scan.exitCode).toBe(0); + expect(scan.stdout).toContain('name="SkillSync scan"'); + expect(scan.stdout).not.toContain(root); +}); diff --git a/tests/docs/documentation.test.ts b/tests/docs/documentation.test.ts index 2dd2f62..db24782 100644 --- a/tests/docs/documentation.test.ts +++ b/tests/docs/documentation.test.ts @@ -38,7 +38,7 @@ describe("release documentation", () => { repository?: { type?: string; url?: string }; }; - expect(readme).toContain("Verify Agent Skills before you trust them."); + expect(readme).toContain("Inspect local Skill folders and declared artifact deliveries."); expect(readme).toContain("Alpha · GitHub release v0.3.0 · Node.js 20+"); expect(readme).toContain("GitHub Releases"); expect(readme).toContain("download/v0.3.0"); diff --git a/tests/reporters/junit.test.ts b/tests/reporters/junit.test.ts new file mode 100644 index 0000000..8657b57 --- /dev/null +++ b/tests/reporters/junit.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import type { Finding } from "../../src/domain/result.js"; +import { renderJUnit } from "../../src/reporters/junit.js"; + +const finding = (status: Finding["status"], changes: Partial = {}): Finding => ({ + level: 2, severity: "error", status, code: "synthetic.check", skill: "example", + message: "Synthetic finding", evidence: [], ...changes, +}); + +describe("JUnit evidence output", () => { + it("keeps findings and policy outcomes distinct, escapes XML and omits raw evidence", () => { + const output = renderJUnit({ findings: [finding("pass"), finding("warn"), finding("unknown"), + finding("fail", { code: 'bad<&"', skill: "/private/example/project", message: "bad & value\u0000\ud800😀", + evidence: [{ raw: "NEVER_REPORT_RAW_SLOT" }] })], exitCode: 1 }, "verify"); + expect(output).toContain('tests="5" failures="2" errors="0" skipped="2"'); + expect(output.match(/ { + expect(renderJUnit({ findings: [] }, "scan")).toContain('tests="1" failures="0" errors="0" skipped="0"'); + expect(renderJUnit({ findings: [], exitCode: 2 }, "verify")).toContain(" renderJUnit({ findings: Array.from({ length: 10001 }, () => finding("pass")) }, "scan")).toThrow(/capacity/i); + expect(() => renderJUnit({ findings: Array.from({ length: 3000 }, () => finding("fail", { message: "x".repeat(2048) })) }, "verify")).toThrow(/capacity/i); + }); +});