diff --git a/.github/workflows/hypatia-scan-reusable.yml b/.github/workflows/hypatia-scan-reusable.yml index 0f1c5ba55..11cadf227 100644 --- a/.github/workflows/hypatia-scan-reusable.yml +++ b/.github/workflows/hypatia-scan-reusable.yml @@ -30,6 +30,13 @@ on: callers pass ONLY this secret instead of `secrets: inherit`, which forwarded every caller secret (standards#1108 §6b). required: false + outputs: + verdict: + description: >- + Overall state of the lane's verdict grid (docs/VERDICT-GRID-SPEC.adoc): + purple, red, orange, yellow, blue, green or grey. Callers that comment + on or gate from the grid read this and the verdict-grid artifact. + value: ${{ jobs.scan.outputs.verdict }} permissions: actions: read @@ -44,12 +51,18 @@ jobs: # at 5, so a slow build or a hung scan records "not_run" or "error" instead # of timing out, and so failing, the whole job. timeout-minutes: 45 + outputs: + verdict: ${{ steps.grid.outputs.state }} env: # panic-attack v2.5.5, built from source at this commit (cargo --rev # checks out exactly this object; --locked holds every dependency to the # Cargo.lock checksums). There is no release binary to pin instead. # The binary reports "2.1.0" for --version; that string is stale upstream. PANIC_ATTACK_SHA: 2dc1393c0ac982dbad98ba07f78a580d319456a5 + # pons asinorum, built from source at this commit the same way. Its + # `scan --format json` envelope carries each finding's tier and + # evidence class, so a heuristic is never read as a proof. + PONS_SHA: 32e55677d8fd1f3010848dee22e822c2f385c2a0 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -372,6 +385,103 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" fi + # pons asinorum is ADVISORY on the same terms as panic-attack: a status + # envelope (ran | not_run | error) that is never a bare pass, built from + # source at PONS_SHA and cached by that commit. + - name: Restore pons build cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.pons + key: pons-v1-${{ runner.os }}-${{ runner.arch }}-${{ env.PONS_SHA }} + + - name: Install pons asinorum (pinned source build) + id: pons-install + run: | + set -euo pipefail + root="$HOME/.pons" + bin="$root/bin/pons" + reason="" + if [ -x "$bin" ] && grep -qF "#$PONS_SHA" "$root/.crates.toml" 2>/dev/null; then + echo "pons restored from cache at $PONS_SHA" + elif ! command -v cargo >/dev/null 2>&1; then + reason="cargo is not available on this runner" + elif ! timeout 600 cargo install --git https://github.com/hyperpolymath/pons-asinorum \ + --rev "$PONS_SHA" --locked --force --root "$root" pons-cli; then + reason="source build at $PONS_SHA failed or exceeded 10 minutes" + elif [ ! -x "$bin" ]; then + reason="no binary after the source build at $PONS_SHA" + fi + if [ -n "$reason" ]; then + rm -rf "$root" + echo "::warning title=pons not installed::$reason" + echo "reason=$reason" >> "$GITHUB_OUTPUT" + else + echo "bin=$bin" >> "$GITHUB_OUTPUT" + fi + + - name: Run pons asinorum scan + id: pons + env: + PONS_BIN: ${{ steps.pons-install.outputs.bin }} + PONS_INSTALL_REASON: ${{ steps.pons-install.outputs.reason }} + run: | + set -euo pipefail + # pons-findings.json is the same status envelope as panic-attack's; + # `report` is pons's own JSON envelope when status is "ran". + out=pons-findings.json + marker() { + jq -n --arg status "$1" --arg reason "$2" --argjson exit_code "$3" \ + --arg ref "$PONS_SHA" '{status: $status, tool: "pons-asinorum", + ref: $ref, reason: $reason, exit_code: $exit_code, counts: null, + report: null}' > "$out" + } + if [ -z "${PONS_BIN:-}" ] || [ ! -x "$PONS_BIN" ]; then + marker not_run "${PONS_INSTALL_REASON:-pons is not installed}" null + else + report=$(mktemp) + log=$(mktemp) + rc=0 + timeout 300 "$PONS_BIN" scan --format json . > "$report" 2> "$log" || rc=$? + if [ "$rc" -eq 124 ]; then + marker error "pons scan did not finish within 5 minutes" "$rc" + elif [ "$rc" -ne 0 ]; then + marker error "pons scan exited $rc" "$rc" + elif ! jq -e -s 'length == 1 and (.[0] | type == "object" and + (.findings | type) == "array" and all(.findings[]; type == "object" and + ((.severity | ascii_upcase) as $s | ["INFO", "WARN", "ERROR"] | index($s) != null)))' \ + "$report" > /dev/null 2>&1; then + marker error "pons scan exited 0 without a valid report" 0 + else + jq --arg ref "$PONS_SHA" ' + [.findings[].severity | ascii_upcase] as $s + | {status: "ran", tool: "pons-asinorum", ref: $ref, reason: null, exit_code: 0, + counts: {error: ($s | map(select(. == "ERROR")) | length), + warn: ($s | map(select(. == "WARN")) | length), + info: ($s | map(select(. == "INFO")) | length)}, + report: .}' "$report" > "$out" + fi + if [ "$(jq -r .status "$out")" = error ]; then + echo "::group::pons output" + cat "$log" + echo "::endgroup::" + fi + fi + status=$(jq -r .status "$out") + echo "status=$status" >> "$GITHUB_OUTPUT" + if [ "$status" = ran ]; then + { + echo "## pons asinorum scan (advisory, does not gate)" + echo "" + echo "| Severity | Count |" + echo "|----------|-------|" + jq -r '"| ERROR | \(.counts.error) |", "| WARN | \(.counts.warn) |", "| INFO | \(.counts.info) |"' "$out" + echo "" + echo "Built from hyperpolymath/pons-asinorum@$PONS_SHA. Each finding carries its tier and evidence class; a heuristic is labelled as one." + } >> "$GITHUB_STEP_SUMMARY" + else + echo "::warning title=pons $status::$(jq -r .reason "$out"). pons-findings.json records status $status, which is not a pass." + fi + - name: Check out standards for the SARIF baseline filter id: fetch_filter # ⚠ continue-on-error is on the CHECKOUT, never on the check. If the @@ -448,6 +558,95 @@ jobs: path: | hypatia-findings.json panic-attack-findings.json + pons-findings.json + retention-days: 90 + + # ── VERDICT GRID (docs/VERDICT-GRID-SPEC.adoc) ────────────────────── + # One table for the lane: Hypatia, panic-attack, pons and every + # gitbot-fleet report-back under .gitbot-fleet/, graded by the worst + # finding present, with missing or unreadable evidence orange and + # explicit exclusions grey. It lands in the job summary, in the + # verdict-grid artifact (report.json, report.md, badge.json) and in the + # `verdict` output. The PR comment is the CALLER's job: this reusable + # declares only the permissions every existing caller already grants, + # and a called workflow may not ask for more than its caller. + - name: Check out the verdict grid renderer at this workflow's commit + if: ${{ always() }} + # continue-on-error on the CHECKOUT only: the render step below + # reports a missing renderer as orange, never as a pass. + continue-on-error: true + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: hyperpolymath/standards + # `job.workflow_sha`: the commit of THIS reusable file, i.e. what the + # caller pinned. See ci-pipeline.yml for why neither + # github.workflow_sha nor github.job_workflow_sha is that. + ref: ${{ job.workflow_sha }} + path: .standards-grid + persist-credentials: false + sparse-checkout: | + scripts/verdict-grid.sh + scripts/lib/verdict-grid.jq + sparse-checkout-cone-mode: false + + - name: Read the previous verdict grid + if: ${{ always() && github.event_name == 'pull_request' }} + # Best effort: the token may lack pull-request read on the caller. A + # missing previous grid only means "no comparable baseline". + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + | jq -r '[.[][] | select(.user.login == "github-actions[bot]" and (.body | contains("' \ + '🟠 indeterminate · the verdict grid renderer could not be checked out from hyperpolymath/standards; see the job log.' \ + > verdict-grid/report.md + cat verdict-grid/report.md >> "$GITHUB_STEP_SUMMARY" + echo "state=orange" >> "$GITHUB_OUTPUT" + exit 0 + fi + findings=hypatia-findings.json + [ -f hypatia-findings.relativized.json ] && findings=hypatia-findings.relativized.json + enrolment="" + [ -f .gitbot-fleet/enrolment.json ] && enrolment=.gitbot-fleet/enrolment.json + # The renderer appends the grid to the step summary and writes + # state= to GITHUB_OUTPUT itself; its exit code is 0 for any + # rendered verdict and 2 only for a usage error. + bash "$renderer" render --out verdict-grid \ + --hypatia "$findings" --panic panic-attack-findings.json --pons pons-findings.json \ + --report-backs .gitbot-fleet/report-backs --enrolment "$enrolment" \ + --previous "$RUNNER_TEMP/grid-previous.md" \ + --sha "$HEAD_SHA" --run "$GITHUB_RUN_ID" \ + --url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + --pr "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --hypatia-ref "$HYPATIA_REF" --panic-ref "$PANIC_ATTACK_SHA" --pons-ref "$PONS_SHA" > /dev/null + + - name: Upload the verdict grid + if: ${{ always() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: verdict-grid + path: verdict-grid + if-no-files-found: error retention-days: 90 - name: Gate on baseline (blocking when a baseline is committed) diff --git a/1-formats/k9/tools/CONFORMANCE-REPORTING.adoc b/1-formats/k9/tools/CONFORMANCE-REPORTING.adoc index e5e86e50f..1fab0478a 100644 --- a/1-formats/k9/tools/CONFORMANCE-REPORTING.adoc +++ b/1-formats/k9/tools/CONFORMANCE-REPORTING.adoc @@ -5,6 +5,12 @@ produces a structured result plus one compact Markdown report. This is local reporting policy, not a new normative contract. It adds no K9 conformance rules. +The report's shape (badge, identity line, grid, accordion, scope footer, +embedded baseline) is the *verdict grid* described in +link:../../../docs/VERDICT-GRID-SPEC.adoc[docs/VERDICT-GRID-SPEC.adoc]; this +lane is its conformance-family reference. The Hypatia lane renders the same +shape with the findings family. + == What each category establishes [cols="1,3,3",options="header"] diff --git a/docs/VERDICT-GRID-SPEC.adoc b/docs/VERDICT-GRID-SPEC.adoc new file mode 100644 index 000000000..1aefcc146 --- /dev/null +++ b/docs/VERDICT-GRID-SPEC.adoc @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) += Verdict Grid (VG) v1.0 — compact PR verdicts for conformance and findings lanes +:toc: left +:toclevels: 3 +:sectnums: +:date: 2026-10-19 + +A verdict grid is the one-screen report a CI lane posts on a pull request: +a badge, an identity line, a table with one row per thing that was checked, +an accordion of detail under it, a scope footer, and an embedded baseline so +the next revision can say what moved. The DEED–K9 lane has rendered one since +standards#1058 (link:../1-formats/k9/tools/CONFORMANCE-REPORTING.adoc[]). +This document names the design so the Hypatia lane, and any gitbot-fleet +member that reports back to a pull request, renders the same shape with the +grading family that fits what it measures. + +It is local reporting policy. It adds no conformance rule, no security +claim and no new readiness grade (link:BADGE-CRITERIA-SPEC.adoc[] §Non-goals +still applies: a grid state is never a fifth grade). + +== Anatomy + +Every grid has exactly these parts, in this order. + +[cols="1,3",options="header"] +|=== +|Part |Content + +|Marker +|An HTML comment naming the lane, e.g. ``. The posting step finds its own previous comment by this marker and updates it in place; one comment per lane per PR. + +|Badge line +|One shields.io static badge: `` – `` – ``. It restates the overall state and nothing else. + +|Identity line +|The overall state label, the commit (12 hex), the run link, the scanner commits where a tool is pinned, and `working tree modified` when rendered locally on a dirty tree. + +|Grid +|One table. Conformance lanes: `Category · Status · Pass · Fail · Indeterminate · N/A · Since previous revision`. Findings lanes: `Reporter · Status · Critical · High · Medium · Minor · Since previous revision`. Counts are results, files or findings, never lines of log. + +|Accordion +|One `
` block per row that has anything to show: every non-passing item or finding, worst first, at most 20, with a count of the rest. A row whose evidence is missing shows its reason here too. + +|Scope footer +|What green means in this lane, what orange and grey mean, and where the full evidence is (the dated artifact on the linked run). + +|Baseline +|`` (`k9-baseline` for the K9 lane, `verdict-grid-baseline` for findings lanes): schema, profile hash, PR, commit, and per-row counts and status. Small, validated on read, ignored when garbled. +|=== + +== Two grading families + +A lane grades either *conformance* (did the thing pass the rules it must +pass?) or *findings* (what did a scanner report, and how bad is the worst of +it?). The two families share orange and grey and nothing else; mixing them +in one row is an error. + +=== Conformance family (DEED–K9 lane) + +[cols="1,1,4",options="header"] +|=== +|State |Colour |Meaning +|red |`red` |At least one blocking conformance failure is known. +|orange |`orange` |Required evidence is incomplete, unavailable, interrupted or malformed — including a validator whose self-test failed. +|yellow |`yellow` |Checks completed with permitted advisory debt or warnings. +|green |`brightgreen` |Every applicable check in the declared scope completed and passed. +|grey |`lightgrey` |Explicitly not applicable, with a reason. Never a synonym for "not run". +|=== + +Precedence: red > orange > yellow > green; grey only when every item is +grey. Red outranks orange so that missing evidence cannot hide a known +failure. Reference rendering: `1-formats/k9/tools/conformance-report.mjs`. + +=== Findings family (Hypatia lane, fleet report-backs) + +[cols="1,1,4",options="header"] +|=== +|State |Colour |Meaning +|purple |`blueviolet` |The reporter ran and its worst finding is *critical*. +|red |`red` |The reporter ran and its worst finding is *high*. +|orange |`orange` |The reporter did not run, errored, produced no readable result, or its self-test failed. Also an enrolled fleet member that sent no report-back. +|yellow |`yellow` |The reporter ran and its worst finding is *medium* (Hypatia's `warn` ranks here). +|blue |`blue` |The reporter ran and reported only *low* or *info* findings: present, minor. +|green |`brightgreen` |The reporter ran and reported nothing. +|grey |`lightgrey` |The reporter is explicitly not relevant to this repository, with a reason (enrolment manifest), or declared itself not applicable in its report-back. +|=== + +Precedence: purple > red > orange > yellow > blue > green; grey only when +every row is grey; an empty grid is orange. As in the conformance family, a +known critical or high finding outranks missing evidence, and missing +evidence outranks anything milder than high: a scanner that did not run is +never reported as a clean scan, and never hides a finding another row +produced. + +Severity words are normalised on input: `critical`; `high` or `error`; +`medium`, `warn` or `warning`; `low`; `info`, `informational`, `note` or +`suggestion`. Anything else makes the whole row orange with the reason +"a finding carries a severity the grid does not recognise". Findings a tool +itself suppresses as likely false positives (panic-attack `suppressed: true`) +are not counted. + +== Rows of a findings lane + +A findings grid has one row per *reporter*. Three are always present in the +Hypatia lane; the rest come from the fleet. + +[cols="1,2,3",options="header"] +|=== +|Row |Evidence read |How it goes orange + +|Hypatia scan +|`hypatia-findings.relativized.json`: one JSON array of findings with `severity`, `rule_module`, `type`, `file`, `line`, `reason`. +|File absent, not JSON, not one array of objects, or a severity the grid does not know. The reusable's "Validate findings" step failing is exactly this case. + +|panic-attack assail +|`panic-attack-findings.json`, the status envelope the reusable already writes: `status` `ran` / `not_run` / `error`, `ref`, `reason`, `report.weak_points[]` with `category`, `file`, `severity`, `description`, optional `suppressed`. +|`status` other than `ran`, or `ran` without a readable `report`. + +|pons asinorum +|`pons-findings.json`, the same envelope around `pons scan --format json` (`report.findings[]` with `rule_id`, `severity` `ERROR`/`WARN`/`INFO`, `tier`, `evidence`, `location.file`, `location.line_start`, `message`). The tier and evidence class are carried into the message, so a heuristic is never read as a proof. +|As above. + +|One row per gitbot-fleet member +|`.gitbot-fleet/report-backs/.json` in the scanned tree, each a report-back envelope (below), and `.gitbot-fleet/enrolment.json` naming which members apply to this repository. +|An enrolled member with no report-back is orange ("enrolled, but no report-back was received"); a member marked not enrolled is grey with the manifest's reason; a member with a report-back but no enrolment entry gets a row from its report-back alone. An unreadable manifest is one orange row named `gitbot-fleet enrolment`. +|=== + +=== The report-back envelope + +A fleet member reports back by writing one JSON object: + +[source,json] +---- +{ "reporter": "rhodibot", "title": "rhodibot (RSR compliance)", + "status": "ran", // ran | not_run | error | not_applicable + "reason": null, // required unless ran + "ref": "", + "findings": [ { "rule_id": "RSR-001", "severity": "error", + "file": "README.adoc", "line": 1, "message": "…" } ] } +---- + +`severity` uses the fleet's own words (`error`, `warning`, `info`, +`suggestion` — `shared-context/src/finding.rs`) or the scanner words above; +both normalise. `not_applicable` must carry a reason and renders grey. + +=== The enrolment manifest + +[source,json] +---- +{ "schema": 1, + "reporters": { + "rhodibot": { "title": "rhodibot (RSR compliance)", "enrolled": true }, + "glambot": { "enrolled": false, "reason": "no served HTML surface" } } } +---- + +Enrolment is a statement by the repository about which members apply. It is +what lets a row be grey instead of orange, and it is hashed into the profile, +so changing it ends comparisons with earlier revisions. + +== Comparison + +"Since previous revision" compares the current grid with the baseline +embedded in the lane's previous comment on the same PR, and only when the +schema and the *profile* match and the commit differs. The profile is a +SHA-256 over the renderer, the enrolment manifest and the pinned tool +commits; a changed tool or scope yields `— no comparable baseline`, never an +apparent improvement. A row whose status changed (a scanner that ran last +time and not this time) reads `— evidence changed`. Rows that ran both times +compare blocking findings (critical + high) first, then medium, then minor: +`↑ improved (n fewer blocking)`, `↓ regressed (n more blocking)`, +`↑ fewer medium`, `↔ unchanged`, and so on. + +== Badges + +Each lane publishes its state three ways, all derived from one `report.json`: + +. In the comment, the shields static badge on the badge line. +. In the artifact, `badge.json` in the shields *endpoint* schema + (`{"schemaVersion":1,"label":…,"message":…,"color":…}`), so a repository + that chooses to publish it (for example to a `governance-badges` branch + on pushes to `main`) can show a live README badge through + `https://img.shields.io/endpoint?url=…`. +. As the job output `verdict` of the reusable workflow, for callers that gate + or comment in their own job. + +A badge asserts the lane's state at one commit and nothing more. It is not +a readiness grade (link:BADGE-CRITERIA-SPEC.adoc[]) and not a security +certification; the scope footer says so in every rendering. + +== Where it runs + +[cols="1,2,2",options="header"] +|=== +|Lane |Renderer |Workflow +|DEED–K9 (conformance) |`1-formats/k9/tools/conformance-report.mjs` (tests: `conformance-report.test.mjs`) |`k9-contractile.yml`, which also posts the comment. +|Hypatia (findings) |`scripts/verdict-grid.sh` + `scripts/lib/verdict-grid.jq` (tests: `scripts/tests/verdict-grid-test.sh`) |`hypatia-scan-reusable.yml` renders the grid into the job summary and the `verdict-grid` artifact and exposes the `verdict` output. It does not post the comment: it declares only the permissions every existing caller already grants, and a reusable may not request more than its caller. The caller posts (see below). +|=== + +=== Posting from a caller + +[source,yaml] +---- +jobs: + hypatia: + uses: hyperpolymath/standards/.github/workflows/hypatia-scan-reusable.yml@ + permissions: { actions: read, contents: read, security-events: write } + secrets: { HYPATIA_SCAN_PAT: ${{ secrets.HYPATIA_SCAN_PAT }} } + verdict: + needs: hypatia + if: always() && github.event_name == 'pull_request' + permissions: { pull-requests: write, contents: read } + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@ # name: verdict-grid + - run: | # find the marked comment, PATCH it, else POST (same pattern as k9-contractile.yml) +---- + +The reusable reads the previous comment only to fill the delta column; that +read is best-effort and a missing previous comment means `— no comparable +baseline`. + +== Reproduce locally + +[source,bash] +---- +bash scripts/tests/verdict-grid-test.sh +HYPATIA_FORMAT=json hypatia scan . --exit-zero > hypatia-findings.json +bash scripts/verdict-grid.sh render --out verdict-grid \ + --hypatia hypatia-findings.json --panic panic-attack-findings.json --pons pons-findings.json \ + --report-backs .gitbot-fleet/report-backs --enrolment .gitbot-fleet/enrolment.json +---- + +Exit 0 means a grid was rendered, whatever its state; exit 2 is a usage +error. Gating is the workflow's decision, taken from `report.json`. + +== Non-goals + +* No composite score across lanes, and no averaging within one: a grid is + worst-of, and two lanes are two comments. +* No change to what any scanner finds. The grid reads results; it does not + filter them. Baseline suppression (`.hypatia-baseline.json`) remains the + gate's business and is not applied to the grid, so the grid shows the + repository's standing debt as well as new findings; the delta column is + what isolates a PR's effect. +* No JavaScript. The findings renderer is bash and jq so it adds nothing to + the estate's authored-JavaScript surface (`docs/NO-JAVASCRIPT-SOURCE-POLICY.adoc`). The K9 lane's `.mjs` renderer predates this + document; migrating it onto `verdict-grid.jq` is follow-up work, not part of this specification. diff --git a/scripts/lib/verdict-grid.jq b/scripts/lib/verdict-grid.jq new file mode 100644 index 000000000..388382209 --- /dev/null +++ b/scripts/lib/verdict-grid.jq @@ -0,0 +1,198 @@ +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# verdict-grid.jq — the findings-family verdict grid (docs/VERDICT-GRID-SPEC.adoc). +# +# Input (one object, assembled by scripts/verdict-grid.sh): +# { sources: [ {kind, name, present, valid, doc} ... ], +# enrolment: object | null, # .gitbot-fleet/enrolment.json, parsed +# previous: object | null, # baseline embedded in the previous comment +# identity: {lane, sha, run, url, pr, repo, dirty, refs: {hypatia, panic, pons}}, +# profile: "sha256 hex" } +# Output: the structured report; `render` turns it into Markdown. +# +# Every reporter is normalised to one record +# { id, title, status: ran|not_run|error|not_applicable, reason, ref, findings } +# and every finding to { id, severity, file, line, message } with severity in +# critical | high | medium | low | info. Anything the adapter does not +# understand becomes status "error" with a reason: unreadable evidence is +# never a clean row. + +def rank: {critical: 5, high: 4, medium: 3, low: 2, info: 1}[.] // 0; + +def norm_sev: + (. // "" | tostring | ascii_downcase) + | if . == "critical" then "critical" + elif . == "high" or . == "error" then "high" + elif . == "medium" or . == "warn" or . == "warning" then "medium" + elif . == "low" then "low" + elif . == "info" or . == "informational" or . == "note" or . == "suggestion" then "info" + else "unknown" end; + +def finding(id; sev; file; line; msg): + { id: (id // "" | tostring), severity: (sev | norm_sev), + file: (file // "" | tostring), line: (line // 0 | tonumber? // 0), + message: (msg // "" | tostring) }; + +def reporter(id; title; status; reason; ref; findings): + { id: id, title: title, status: status, reason: (reason // ""), ref: (ref // ""), findings: findings } + | if .status == "ran" and (.findings | any(.severity == "unknown")) + then .status = "error" | .reason = "a finding carries a severity the grid does not recognise" + else . end; + +# A status envelope shared by panic-attack and pons: +# {status: ran|not_run|error, tool, ref, reason, exit_code, counts, report} +def envelope(id; title; src; items): + if src.present | not then reporter(id; title; "not_run"; "no status envelope was produced"; ""; []) + elif src.valid | not then reporter(id; title; "error"; "the status envelope is not valid JSON"; ""; []) + elif (src.doc | type) != "object" or (src.doc.status | type) != "string" + then reporter(id; title; "error"; "the status envelope has no status"; ""; []) + elif src.doc.status == "ran" then + (try (src.doc | items) catch null) as $f + | if $f == null then reporter(id; title; "error"; "the envelope reports a run but carries no readable report"; src.doc.ref; []) + else reporter(id; title; "ran"; ""; src.doc.ref; $f) end + elif src.doc.status == "not_run" or src.doc.status == "error" + then reporter(id; title; src.doc.status; (src.doc.reason // "no reason recorded"); src.doc.ref; []) + else reporter(id; title; "error"; "unrecognised envelope status \(src.doc.status)"; src.doc.ref; []) end; + +def hypatia(src; ref): + if src.present | not then reporter("hypatia"; "Hypatia scan"; "error"; "the scanner produced no findings array"; ref; []) + elif src.valid | not then reporter("hypatia"; "Hypatia scan"; "error"; "the findings file is not valid JSON"; ref; []) + elif (src.doc | type) != "array" or (src.doc | all(type == "object")) | not + then reporter("hypatia"; "Hypatia scan"; "error"; "the findings file is not one array of objects"; ref; []) + else reporter("hypatia"; "Hypatia scan"; "ran"; ""; ref; + [src.doc[] | finding("\(.rule_module // "?")/\(.type // "?")"; .severity; .file; .line; .reason)]) end; + +def panic(src): + envelope("panic-attack"; "panic-attack assail"; src; + [.report.weak_points[] | select(.suppressed != true) + | finding(.category; .severity; .file; 0; .description)]); + +def pons(src): + envelope("pons-asinorum"; "pons asinorum"; src; + [.report.findings[] + | finding(.rule_id; .severity; .location.file; .location.line_start; + "\(.message) (\(.tier // "?"), \(.evidence // "?"))")]); + +def reportback(src): + (src.name) as $id + | if src.valid | not then reporter($id; $id; "error"; "the report-back is not valid JSON"; ""; []) + elif (src.doc | type) != "object" then reporter($id; $id; "error"; "the report-back is not an object"; ""; []) + else (src.doc.reporter // $id) as $rid + | (src.doc.title // $rid) as $title + | if src.doc.status == "ran" then + reporter($rid; $title; "ran"; ""; src.doc.ref; + [ (src.doc.findings // [])[] | finding(.rule_id // .id; .severity; .file; .line; .message) ]) + elif (src.doc.status | IN("not_run", "error", "not_applicable")) + then reporter($rid; $title; src.doc.status; (src.doc.reason // "no reason recorded"); src.doc.ref; []) + else reporter($rid; $title; "error"; "unrecognised report-back status \(src.doc.status)"; src.doc.ref; []) end + end; + +def counts: + { critical: ([.findings[] | select(.severity == "critical")] | length), + high: ([.findings[] | select(.severity == "high")] | length), + medium: ([.findings[] | select(.severity == "medium")] | length), + minor: ([.findings[] | select(.severity == "low" or .severity == "info")] | length) }; + +def state: + if .status == "not_applicable" then "grey" + elif .status != "ran" then "orange" + else ([.findings[].severity | rank] | max // 0) as $m + | if $m >= 5 then "purple" elif $m == 4 then "red" elif $m == 3 then "yellow" + elif $m >= 1 then "blue" else "green" end end; + +# Precedence: a known critical beats missing evidence, which beats anything milder. +def overall: + if length == 0 then "orange" + elif all(. == "grey") then "grey" + else (["purple", "red", "orange", "yellow", "blue", "green"] as $order + | . as $states | $order | map(select(. as $c | $states | index($c))) | first) end; + +def build: + . as $in + | ($in.sources | map(select(.kind == "hypatia")) | first // {present: false}) as $h + | ($in.sources | map(select(.kind == "panic")) | first // {present: false}) as $p + | ($in.sources | map(select(.kind == "pons")) | first // {present: false}) as $q + | [ hypatia($h; $in.identity.refs.hypatia), panic($p), pons($q) ] as $builtin + | [ $in.sources[] | select(.kind == "report-back") | reportback(.) ] as $backs + | ($in.enrolment.reporters // {}) as $enrol + | [ $enrol | to_entries[] | .key as $id | .value as $e + | select(($backs | map(.id) | index($id)) == null) + | if $e.enrolled == true + then reporter($id; ($e.title // $id); "not_run"; "enrolled, but no report-back was received"; ""; []) + else reporter($id; ($e.title // $id); "not_applicable"; ($e.reason // "not enrolled for this repository"); ""; []) end ] as $missing + | ($builtin + $backs + $missing + | map(. + {counts: counts, state: state})) as $reporters + | { schema: 1, lane: ($in.identity.lane // "hypatia"), profile: $in.profile, + sha: ($in.identity.sha // ""), run: ($in.identity.run // "local"), url: ($in.identity.url // ""), + pr: ($in.identity.pr // "local"), repo: ($in.identity.repo // ""), dirty: ($in.identity.dirty // false), + generated: ($in.identity.generated // ""), refs: ($in.identity.refs // {}), + reporters: $reporters, + state: ($reporters | map(.state) | overall), + counts: ({findings: $reporters} | counts) }; + +# One row's movement since the previous revision of the same PR. `report` is +# the whole report (profile, pr, sha); `.` is the row. +def delta(prev; report): + . as $r + | if prev == null or prev.schema != 1 or prev.profile != report.profile or prev.pr != report.pr or prev.sha == report.sha + then "— no comparable baseline" else . end + | if type == "string" then . else + (prev.reporters | map(select(.id == $r.id)) | first) as $b + | if $b == null then "— no comparable baseline" + elif $b.status != $r.status then "— evidence changed" + elif $r.status == "not_applicable" then "↔ unchanged" + elif $r.status != "ran" then "— indeterminate" + else (($b.counts.critical + $b.counts.high) as $a | ($r.counts.critical + $r.counts.high) as $c + | if $c < $a then "↑ improved (\($a - $c) fewer blocking)" + elif $c > $a then "↓ regressed (\($c - $a) more blocking)" + elif $r.counts.medium < $b.counts.medium then "↑ fewer medium" + elif $r.counts.medium > $b.counts.medium then "↓ more medium" + elif $r.counts.minor < $b.counts.minor then "↑ fewer minor" + elif $r.counts.minor > $b.counts.minor then "↓ more minor" + else "↔ unchanged" end) end end; + +def state_label: {purple: "🟣 critical", red: "🔴 high", orange: "🟠 indeterminate", yellow: "🟡 medium", + blue: "🔵 minor", green: "🟢 clean", grey: "⚪ not applicable"}[.]; +def colour: {purple: "blueviolet", red: "red", orange: "orange", yellow: "yellow", + blue: "blue", green: "brightgreen", grey: "lightgrey"}[.]; +def word: {purple: "critical", red: "high", orange: "indeterminate", yellow: "medium", + blue: "minor", green: "clean", grey: "not_applicable"}[.]; +def esc: tostring | gsub("&"; "&") | gsub("<"; "<") | gsub(">"; ">") | gsub("\\|"; "|") + | gsub("`"; "`") | gsub("\r?\n"; " "); +def badge_url(lbl; st): "https://img.shields.io/badge/\(lbl | @uri)-\(st | word | @uri)-\(st | colour)"; + +def render(prev): + . as $r + | ($r.lane | if . == "hypatia" then "Hypatia grid" else . end) as $lbl + | "\n" + + "![\($lbl) \($r.state)](\(badge_url($lbl; $r.state)))\n\n" + + "\($r.state | state_label) · commit `\($r.sha[0:12] | esc)` · " + + (if $r.url != "" then "[run \($r.run | esc)](\($r.url))" else "local run" end) + + (if $r.dirty then " · working tree modified" else "" end) + "\n\n" + + "| Reporter | Status | Critical | High | Medium | Minor | Since previous revision |\n|---|---|---:|---:|---:|---:|---|\n" + + ([ $r.reporters[] + | "| \(.title | esc) | \(.state | state_label) | \(.counts.critical) | \(.counts.high) | \(.counts.medium) | \(.counts.minor) | \(delta(prev; $r)) |\n" ] | add // "") + + ([ $r.reporters[] + | select(.status != "ran" or (.findings | length) > 0) + | "\n
\n\(.title | esc): " + + (if .status == "ran" then "\(.findings | length) finding(s)" else "\(.status | esc) — \(.reason | esc)" end) + + "\n\n" + + (if .status != "ran" then "- **\(.state | state_label)**: \(.reason | esc)\n" else "" end) + + ([ .findings | sort_by(.severity | rank) | reverse | .[0:20][] + | "- **\(.severity | esc)** `\(.id | esc)` — \(if .file != "" then "`\(.file | esc)\(if .line > 0 then ":\(.line)" else "" end)` — " else "" end)\(.message | esc | .[0:280])\n" ] | add // "") + + (if (.findings | length) > 20 then "- \((.findings | length) - 20) more; see the structured report in the artifact.\n" else "" end) + + "\n
\n" ] | add // "") + + "\nScope: static findings reported by each row's tool at the named commit. Purple, red, yellow and blue grade the worst finding present; orange is evidence missing, malformed or a scanner that did not run; grey is an explicit scope exclusion with its reason. No row is a security certification. report.json, badge.json and the raw findings are in the verdict-grid artifact on the linked run.\n" + + "\n\n"; + +def badge: {schemaVersion: 1, label: (.lane | if . == "hypatia" then "Hypatia grid" else . end), + message: (.state | word | gsub("_"; " ")), color: (.state | colour)}; + +# Entry point: $mode selects build (input: the assembled sources object), +# render (input: report.json; $previous: the prior baseline or null) or badge. +if $mode == "build" then build +elif $mode == "render" then render($previous) +elif $mode == "badge" then badge +else error("unknown mode \($mode)") end diff --git a/scripts/tests/verdict-grid-test.sh b/scripts/tests/verdict-grid-test.sh new file mode 100644 index 000000000..a4d87e0ef --- /dev/null +++ b/scripts/tests/verdict-grid-test.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# verdict-grid-test.sh — the findings-family grid never turns missing or +# unreadable evidence into a clean row, grades the worst finding present, +# keeps explicit scope exclusions grey, and compares revisions only under one +# profile. docs/VERDICT-GRID-SPEC.adoc is the contract under test. +# +# Run: bash scripts/tests/verdict-grid-test.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +GRID="$SCRIPT_DIR/../verdict-grid.sh" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +pass=0 +fail=0 +check() { + local label="$1" expected="$2" got="$3" + if [ "$got" = "$expected" ]; then + echo "PASS: $label ($got)"; pass=$((pass + 1)) + else + echo "FAIL: $label expected=$expected got=$got"; fail=$((fail + 1)) + fi +} +render() { bash "$GRID" render --out "$1" "${@:2}" >/dev/null; } +state() { jq -r .state "$1/report.json"; } +row() { jq -r --arg id "$2" '.reporters[] | select(.id == $id) | .state' "$1/report.json"; } + +hyp_clean="$WORK/hyp-clean.json"; echo '[]' > "$hyp_clean" +hyp_crit="$WORK/hyp-crit.json"; echo '[{"severity":"critical","rule_module":"cicd_rules","type":"banned_language_file","file":"a.ts","reason":"x"},{"severity":"warn","rule_module":"r","type":"t","file":"b","reason":"y"}]' > "$hyp_crit" +hyp_med="$WORK/hyp-med.json"; echo '[{"severity":"medium","rule_module":"r","type":"t","file":"b","reason":"y"}]' > "$hyp_med" +hyp_info="$WORK/hyp-info.json"; echo '[{"severity":"info","rule_module":"r","type":"t","file":"b","reason":"y"}]' > "$hyp_info" +hyp_bad="$WORK/hyp-bad.json"; echo 'not json' > "$hyp_bad" +hyp_odd="$WORK/hyp-odd.json"; echo '[{"severity":"catastrophic","rule_module":"r","type":"t","reason":"y"}]' > "$hyp_odd" +pan_ran="$WORK/pan-ran.json"; echo '{"status":"ran","tool":"panic-attack","ref":"abc","reason":null,"exit_code":0,"report":{"weak_points":[{"category":"ProofDrift","file":"t.jl","severity":"Medium","description":"d"},{"category":"X","file":"s","severity":"Critical","description":"e","suppressed":true}]}}' > "$pan_ran" +pan_not="$WORK/pan-not.json"; echo '{"status":"not_run","tool":"panic-attack","ref":"abc","reason":"cargo is not available","exit_code":null,"counts":null,"report":null}' > "$pan_not" +pan_clean="$WORK/pan-clean.json"; echo '{"status":"ran","tool":"panic-attack","ref":"abc","reason":null,"exit_code":0,"report":{"weak_points":[]}}' > "$pan_clean" +pons_ran="$WORK/pons-ran.json"; echo '{"status":"ran","tool":"pons-asinorum","ref":"def","reason":null,"exit_code":0,"report":{"findings":[{"rule_id":"unreachable-after-jump","tier":"T0","evidence":"HEURISTIC","severity":"WARN","location":{"file":"a.tsx","line_start":4},"message":"m"}]}}' > "$pons_ran" +pons_clean="$WORK/pons-clean.json"; echo '{"status":"ran","tool":"pons-asinorum","ref":"def","reason":null,"exit_code":0,"report":{"findings":[]}}' > "$pons_clean" +pons_hollow="$WORK/pons-hollow.json"; echo '{"status":"ran","tool":"pons-asinorum","ref":"def"}' > "$pons_hollow" + +# 1. No evidence at all is orange, never green, and the badge says so. +render "$WORK/o1" +check "no evidence is indeterminate" orange "$(state "$WORK/o1")" +check "badge colour follows the state" orange "$(jq -r .color "$WORK/o1/badge.json")" +check "missing scanner output is an error row" error "$(jq -r '.reporters[] | select(.id == "hypatia") | .status' "$WORK/o1/report.json")" + +# 2. A known critical beats every other state, including missing evidence. +render "$WORK/o2" --hypatia "$hyp_crit" --panic "$pan_not" +check "critical finding is purple" purple "$(state "$WORK/o2")" +check "warn counts as medium" 1 "$(jq '.reporters[0].counts.medium' "$WORK/o2/report.json")" +check "not_run envelope is orange" orange "$(row "$WORK/o2" panic-attack)" +check "badge message names the worst grade" critical "$(jq -r .message "$WORK/o2/badge.json")" + +# 3. Malformed or unrecognised evidence is orange. +render "$WORK/o3" --hypatia "$hyp_bad" --panic "$pan_clean" --pons "$pons_clean" +check "unparseable findings are indeterminate" orange "$(state "$WORK/o3")" +render "$WORK/o3b" --hypatia "$hyp_odd" --panic "$pan_clean" --pons "$pons_clean" +check "unknown severity is indeterminate" orange "$(row "$WORK/o3b" hypatia)" +render "$WORK/o3c" --hypatia "$hyp_clean" --panic "$pan_clean" --pons "$pons_hollow" +check "a ran envelope without a report is indeterminate" orange "$(row "$WORK/o3c" pons-asinorum)" + +# 4. Grades below critical: red, yellow, blue, green; suppressed points are not counted. +render "$WORK/o4" --hypatia "$hyp_med" --panic "$pan_ran" --pons "$pons_ran" +check "medium is yellow" yellow "$(state "$WORK/o4")" +check "suppressed weak points are not counted" 0 "$(jq '.reporters[] | select(.id == "panic-attack") | .counts.critical' "$WORK/o4/report.json")" +check "pons WARN is medium" yellow "$(row "$WORK/o4" pons-asinorum)" +render "$WORK/o5" --hypatia "$hyp_info" --panic "$pan_clean" --pons "$pons_clean" +check "info only is blue" blue "$(state "$WORK/o5")" +render "$WORK/o6" --hypatia "$hyp_clean" --panic "$pan_clean" --pons "$pons_clean" +check "all clean is green" green "$(state "$WORK/o6")" + +# 5. Fleet rows: enrolled-without-report is orange, not-enrolled is grey with a reason, +# a report-back supplies its own row, and grey never masks a finding. +mkdir -p "$WORK/backs" +echo '{"reporter":"rhodibot","title":"rhodibot (RSR)","status":"ran","ref":"r1","findings":[{"rule_id":"RSR-001","severity":"error","file":"README.adoc","message":"missing"}]}' > "$WORK/backs/rhodibot.json" +echo '{"reporters":{"rhodibot":{"enrolled":true},"panicbot":{"title":"panicbot","enrolled":true},"glambot":{"enrolled":false,"reason":"no served HTML"}}}' > "$WORK/enrol.json" +render "$WORK/o7" --hypatia "$hyp_clean" --panic "$pan_clean" --pons "$pons_clean" --report-backs "$WORK/backs" --enrolment "$WORK/enrol.json" +check "fleet error severity is high" red "$(row "$WORK/o7" rhodibot)" +check "enrolled member without a report-back is orange" orange "$(row "$WORK/o7" panicbot)" +check "member not enrolled is grey" grey "$(row "$WORK/o7" glambot)" +check "grey rows do not hide a red one" red "$(state "$WORK/o7")" +check "the grey reason is shown" yes "$(grep -q 'no served HTML' "$WORK/o7/report.md" && echo yes)" +echo 'nonsense' > "$WORK/enrol-bad.json" +render "$WORK/o8" --hypatia "$hyp_clean" --panic "$pan_clean" --pons "$pons_clean" --enrolment "$WORK/enrol-bad.json" 2>/dev/null +check "an unreadable enrolment manifest is missing evidence" orange "$(state "$WORK/o8")" + +# 6. Escaping: untrusted text cannot break the table or inject markup. +echo '[{"severity":"high","rule_module":"r","type":"t
","file":"a|b","reason":"line one\nline two"}]' > "$WORK/hyp-esc.json" +render "$WORK/o9" --hypatia "$WORK/hyp-esc.json" --panic "$pan_clean" --pons "$pons_clean" +check "markup is escaped" 0 "$(grep -c '