From 67e1ca1b4b3bb4822635f17929de7bcb33272815 Mon Sep 17 00:00:00 2001 From: brandom Date: Mon, 10 Aug 2026 12:27:51 -0700 Subject: [PATCH 1/3] Add run-scoped validation report consumer Consume versioned normalized sample artifacts in a separate reporting job and publish an incomplete summary before failing malformed handoffs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/render-validation-report.py | 234 ++++++++++++++++++ .../test/test-render-validation-report.py | 95 +++++++ .github/workflows/scripts-selftest.yml | 9 + .github/workflows/validation-report.yml | 62 +++++ 4 files changed, 400 insertions(+) create mode 100644 .github/scripts/render-validation-report.py create mode 100644 .github/scripts/test/test-render-validation-report.py create mode 100644 .github/workflows/validation-report.yml diff --git a/.github/scripts/render-validation-report.py b/.github/scripts/render-validation-report.py new file mode 100644 index 000000000..0b971a7da --- /dev/null +++ b/.github/scripts/render-validation-report.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Render a run-scoped GitHub Actions summary from normalized result artifacts. + +The artifact shape consumed here is an explicit adapter boundary for the reporting +pilot. The validation session owns the canonical producer schema; this consumer +must be aligned to that schema before production use. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import quote, urlparse + + +OUTCOMES = { + "passed": "✅ Passed", + "sample_failure": "❌ Sample failure", + "infrastructure_error": "⚠️ Infrastructure/error", + "skipped": "⏭️ Skipped/not-completed", +} +REQUIRED_FIELDS = { + "sample", + "outcome", + "stage", + "duration_seconds", + "completed_at", +} + + +class ContractError(ValueError): + """Raised when the reporting adapter input is incomplete or malformed.""" + + +def load_json(path: Path, label: str) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise ContractError(f"{label} not found: {path}") from exc + except json.JSONDecodeError as exc: + raise ContractError(f"{label} is not valid JSON: {exc}") from exc + + +def validate_sample(value: Any, field: str = "sample") -> str: + if not isinstance(value, str) or not value.startswith("samples/"): + raise ContractError(f"{field} must be a repository-relative samples/ path") + if ".." in Path(value).parts or any(c in value for c in "|\r\n"): + raise ContractError(f"{field} contains an unsafe path") + return Path(value).as_posix() + + +def validate_url(value: Any, field: str) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise ContractError(f"{field} must be a string") + parsed = urlparse(value) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + raise ContractError(f"{field} must be an absolute HTTP(S) URL") + return value + + +def parse_timestamp(value: Any, field: str) -> datetime: + if not isinstance(value, str) or not value.endswith("Z"): + raise ContractError(f"{field} must be an ISO-8601 UTC timestamp ending in Z") + try: + parsed = datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError as exc: + raise ContractError(f"{field} is not a valid timestamp") from exc + if parsed.utcoffset() != timezone.utc.utcoffset(parsed): + raise ContractError(f"{field} must be UTC") + return parsed + + +def load_expected(path: Path) -> list[str]: + value = load_json(path, "expected samples") + if not isinstance(value, list) or not value: + raise ContractError("expected samples must be a non-empty JSON array") + samples = [validate_sample(item, "expected sample") for item in value] + if samples != sorted(set(samples)): + raise ContractError("expected samples must be sorted and unique") + return samples + + +def load_record(path: Path) -> dict[str, Any]: + value = load_json(path, f"result artifact {path.name}") + if not isinstance(value, dict): + raise ContractError(f"result artifact {path.name} must be a JSON object") + if value.get("schema_version") != 1: + raise ContractError(f"result artifact {path.name} must use schema_version 1") + missing = REQUIRED_FIELDS - value.keys() + if missing: + raise ContractError( + f"result artifact {path.name} is missing fields: {sorted(missing)}" + ) + sample = validate_sample(value["sample"]) + outcome = value["outcome"] + if outcome not in OUTCOMES: + raise ContractError(f"{sample}.outcome is unsupported: {outcome!r}") + if not isinstance(value["stage"], str) or not value["stage"]: + raise ContractError(f"{sample}.stage must be a non-empty string") + if ( + not isinstance(value["duration_seconds"], (int, float)) + or isinstance(value["duration_seconds"], bool) + or value["duration_seconds"] < 0 + ): + raise ContractError(f"{sample}.duration_seconds must be non-negative") + completed_at = parse_timestamp(value["completed_at"], f"{sample}.completed_at") + diagnostic_url = validate_url(value.get("diagnostic_url"), f"{sample}.diagnostic_url") + artifact_url = validate_url(value.get("artifact_url"), f"{sample}.artifact_url") + return { + "sample": sample, + "outcome": outcome, + "stage": value["stage"], + "duration_seconds": value["duration_seconds"], + "completed_at": completed_at, + "diagnostic_url": diagnostic_url, + "artifact_url": artifact_url, + } + + +def collect_records(results_dir: Path, expected: list[str]) -> tuple[list[dict[str, Any]], bool]: + if not results_dir.is_dir(): + raise ContractError(f"result artifact directory not found: {results_dir}") + records: dict[str, dict[str, Any]] = {} + incomplete = False + for path in sorted(results_dir.glob("*.json")): + try: + record = load_record(path) + except ContractError as exc: + incomplete = True + records[f""] = { + "sample": f"", + "outcome": "infrastructure_error", + "stage": "reporting", + "duration_seconds": 0, + "completed_at": None, + "diagnostic_url": None, + "artifact_url": None, + "error": str(exc), + } + continue + if record["sample"] in records: + incomplete = True + record["error"] = f"duplicate result artifact for {record['sample']}" + record["outcome"] = "infrastructure_error" + records[record["sample"]] = record + + for sample in expected: + if sample not in records: + incomplete = True + records[sample] = { + "sample": sample, + "outcome": "infrastructure_error", + "stage": "reporting", + "duration_seconds": 0, + "completed_at": None, + "diagnostic_url": None, + "artifact_url": None, + "error": "expected result artifact is missing", + } + return sorted(records.values(), key=lambda record: record["sample"]), incomplete + + +def link(value: str | None) -> str: + if not value: + return "—" + encoded = quote(value, safe=":/?#@!$&'*+,;=%._~-") + return f"[link]({encoded})" + + +def render(records: list[dict[str, Any]], run_url: str | None) -> str: + lines = [ + "## Validation report", + "", + "_Run-scoped summary; only attempted samples are listed._", + "", + "| Sample | Outcome | Completed stage | Duration | Last run (UTC) | Diagnostic/artifact |", + "|---|---|---|---:|---|---|", + ] + for record in records: + completed = ( + record["completed_at"].strftime("%Y-%m-%d %H:%M:%S UTC") + if record["completed_at"] + else "—" + ) + evidence = link(record["diagnostic_url"] or record["artifact_url"]) + sample = f"`{record['sample']}`" + lines.append( + f"| {sample} | {OUTCOMES[record['outcome']]} | {record['stage']} | " + f"{record['duration_seconds']}s | {completed} | {evidence} |" + ) + if record.get("error"): + lines.append(f"| `{record['sample']}` | ⚠️ Incomplete | reporting | — | — | {record['error']} |") + if run_url: + lines.extend(["", f"Run evidence: {link(run_url)}"]) + lines.extend( + [ + "", + "**Legend:** ✅ passed · ❌ sample failure · ⚠️ infrastructure/error · " + "⏭️ skipped/not-completed", + "", + ] + ) + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results-dir", type=Path, required=True) + parser.add_argument("--expected-samples", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--run-url") + args = parser.parse_args() + try: + expected = load_expected(args.expected_samples) + records, incomplete = collect_records(args.results_dir, expected) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(render(records, args.run_url), encoding="utf-8", newline="\n") + except (ContractError, OSError) as exc: + print(f"render-validation-report: {exc}", file=sys.stderr) + return 1 + if incomplete: + print("render-validation-report: incomplete result handoff", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/test/test-render-validation-report.py b/.github/scripts/test/test-render-validation-report.py new file mode 100644 index 000000000..21f8e9e9d --- /dev/null +++ b/.github/scripts/test/test-render-validation-report.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).resolve().parents[1] / "render-validation-report.py" +SAMPLE_A = "samples/python/quickstart/a" +SAMPLE_B = "samples/csharp/quickstart/b" + + +class ReportTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.results = self.root / "results" + self.results.mkdir() + self.expected = self.root / "expected.json" + self.expected.write_text(json.dumps(sorted([SAMPLE_A, SAMPLE_B])), encoding="utf-8") + self.output = self.root / "summary.md" + + def tearDown(self) -> None: + self.temp.cleanup() + + def run_report(self) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--results-dir", + str(self.results), + "--expected-samples", + str(self.expected), + "--output", + str(self.output), + "--run-url", + "https://github.com/example/repo/actions/runs/42", + ], + capture_output=True, + text=True, + ) + + def write_result(self, sample: str, outcome: str = "passed") -> None: + name = sample.rsplit("/", 1)[-1] + ".json" + (self.results / name).write_text( + json.dumps( + { + "schema_version": 1, + "sample": sample, + "outcome": outcome, + "stage": "L3", + "duration_seconds": 12.5, + "completed_at": "2026-08-10T19:22:33Z", + "diagnostic_url": "https://github.com/example/repo/actions/runs/42", + } + ), + encoding="utf-8", + ) + + def test_renders_all_outcomes_and_run_freshness(self) -> None: + self.write_result(SAMPLE_A, "passed") + self.write_result(SAMPLE_B, "sample_failure") + completed = self.run_report() + self.assertEqual(completed.returncode, 0, completed.stderr) + body = self.output.read_text(encoding="utf-8") + self.assertIn("✅ Passed", body) + self.assertIn("❌ Sample failure", body) + self.assertIn("2026-08-10 19:22:33 UTC", body) + self.assertIn("Run evidence:", body) + self.assertEqual(body.count("`samples/"), 2) + + def test_missing_expected_artifact_publishes_partial_summary_and_fails(self) -> None: + self.write_result(SAMPLE_A) + completed = self.run_report() + self.assertEqual(completed.returncode, 1) + body = self.output.read_text(encoding="utf-8") + self.assertIn("expected result artifact is missing", body) + self.assertIn("⚠️ Infrastructure/error", body) + + def test_malformed_artifact_publishes_error_row_and_fails(self) -> None: + (self.results / "bad.json").write_text("{", encoding="utf-8") + completed = self.run_report() + self.assertEqual(completed.returncode, 1) + body = self.output.read_text(encoding="utf-8") + self.assertIn("invalid artifact: bad.json", body) + self.assertIn("⚠️ Infrastructure/error", body) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/scripts-selftest.yml b/.github/workflows/scripts-selftest.yml index 9a799b5f7..da4f0af59 100644 --- a/.github/workflows/scripts-selftest.yml +++ b/.github/workflows/scripts-selftest.yml @@ -22,6 +22,7 @@ on: - '.github/scripts/**' - '.github/workflows/validate.yml' - '.github/workflows/scripts-selftest.yml' + - '.github/workflows/validation-report.yml' permissions: contents: read @@ -100,6 +101,14 @@ jobs: - name: Phase-1 exit gate — all 5 languages, pass(0)/fail(1)/error(2) run: bash .github/scripts/test/run-tests.sh + # --- Reporting consumer contract tests: fixtures only, no producer coupling ---------------------- + report-harness: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run normalized-artifact reporting tests + run: python .github/scripts/test/test-render-validation-report.py + # --- Plumbing isolation proof: detector outputs cross a REAL job boundary via needs.*.outputs --- # Proves the $GITHUB_OUTPUT -> steps.*.outputs -> job.outputs -> downstream needs.*.outputs chain # in ISOLATION from validate.yml's trust/gate logic (which only exercises it on trusted gated PRs). diff --git a/.github/workflows/validation-report.yml b/.github/workflows/validation-report.yml new file mode 100644 index 000000000..4b6eb240a --- /dev/null +++ b/.github/workflows/validation-report.yml @@ -0,0 +1,62 @@ +name: validation report + +# Reusable consumer job. The validation producer owns the artifact schema and must +# upload one versioned normalized JSON result per attempted sample plus this sorted +# expected-samples JSON file. This workflow never reads raw logs or language internals. +on: + workflow_call: + inputs: + results-artifact: + required: true + type: string + expected-samples-artifact: + required: true + type: string + +permissions: + contents: read + +jobs: + report: + runs-on: ubuntu-latest + if: ${{ !cancelled() }} + steps: + - uses: actions/checkout@v4 + + - name: Download normalized result artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.results-artifact }} + path: ${{ runner.temp }}/validation-results + + - name: Download attempted sample list + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.expected-samples-artifact }} + path: ${{ runner.temp }}/validation-expected + + - name: Render run-scoped summary + id: render + if: always() + run: | + set -uo pipefail + python .github/scripts/render-validation-report.py \ + --results-dir "$RUNNER_TEMP/validation-results" \ + --expected-samples "$RUNNER_TEMP/validation-expected/expected-samples.json" \ + --output "$RUNNER_TEMP/validation-report.md" \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + rc=$? + cat "$RUNNER_TEMP/validation-report.md" >> "$GITHUB_STEP_SUMMARY" + echo "render_rc=$rc" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Fail incomplete reporting handoff + if: always() + env: + RENDER_RC: ${{ steps.render.outputs.render_rc }} + run: | + set -euo pipefail + if [ "$RENDER_RC" != "0" ]; then + echo "::error::Validation reporting handoff was incomplete or malformed." + exit 1 + fi From 9c652c3e20b1c60c77674dca601f4fa1229573dc Mon Sep 17 00:00:00 2001 From: brandom Date: Mon, 10 Aug 2026 14:15:25 -0700 Subject: [PATCH 2/3] Align report consumer with validation pilot v1 Consume the frozen nested sample and run contract, require diagnostics, and preserve partial-summary failure behavior. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/render-validation-report.py | 232 +++++++----------- .../test/test-render-validation-report.py | 48 +++- .github/workflows/validation-report.yml | 11 +- 3 files changed, 132 insertions(+), 159 deletions(-) diff --git a/.github/scripts/render-validation-report.py b/.github/scripts/render-validation-report.py index 0b971a7da..3eed7ffb5 100644 --- a/.github/scripts/render-validation-report.py +++ b/.github/scripts/render-validation-report.py @@ -1,10 +1,5 @@ #!/usr/bin/env python3 -"""Render a run-scoped GitHub Actions summary from normalized result artifacts. - -The artifact shape consumed here is an explicit adapter boundary for the reporting -pilot. The validation session owns the canonical producer schema; this consumer -must be aligned to that schema before production use. -""" +"""Render a run-scoped summary from validation-pilot v1 result artifacts.""" from __future__ import annotations @@ -14,26 +9,22 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any -from urllib.parse import quote, urlparse - OUTCOMES = { "passed": "✅ Passed", - "sample_failure": "❌ Sample failure", - "infrastructure_error": "⚠️ Infrastructure/error", - "skipped": "⏭️ Skipped/not-completed", + "sample failure": "❌ Sample failure", + "infrastructure/error": "⚠️ Infrastructure/error", + "skipped/not-completed": "⏭️ Skipped/not-completed", } -REQUIRED_FIELDS = { - "sample", - "outcome", - "stage", - "duration_seconds", - "completed_at", +REQUIRED = { + "schema_version", "sample", "outcome", "completed_stage", "duration_seconds", + "diagnostic_reference", "artifact_reference", "completed_at", "run", } +RUN_FIELDS = {"repository", "workflow", "run_id", "run_attempt", "sha", "ref", "started_at"} class ContractError(ValueError): - """Raised when the reporting adapter input is incomplete or malformed.""" + pass def load_json(path: Path, label: str) -> Any: @@ -45,26 +36,7 @@ def load_json(path: Path, label: str) -> Any: raise ContractError(f"{label} is not valid JSON: {exc}") from exc -def validate_sample(value: Any, field: str = "sample") -> str: - if not isinstance(value, str) or not value.startswith("samples/"): - raise ContractError(f"{field} must be a repository-relative samples/ path") - if ".." in Path(value).parts or any(c in value for c in "|\r\n"): - raise ContractError(f"{field} contains an unsafe path") - return Path(value).as_posix() - - -def validate_url(value: Any, field: str) -> str | None: - if value is None: - return None - if not isinstance(value, str): - raise ContractError(f"{field} must be a string") - parsed = urlparse(value) - if parsed.scheme not in ("http", "https") or not parsed.netloc: - raise ContractError(f"{field} must be an absolute HTTP(S) URL") - return value - - -def parse_timestamp(value: Any, field: str) -> datetime: +def timestamp(value: Any, field: str) -> datetime: if not isinstance(value, str) or not value.endswith("Z"): raise ContractError(f"{field} must be an ISO-8601 UTC timestamp ending in Z") try: @@ -76,136 +48,109 @@ def parse_timestamp(value: Any, field: str) -> datetime: return parsed -def load_expected(path: Path) -> list[str]: - value = load_json(path, "expected samples") - if not isinstance(value, list) or not value: - raise ContractError("expected samples must be a non-empty JSON array") - samples = [validate_sample(item, "expected sample") for item in value] - if samples != sorted(set(samples)): - raise ContractError("expected samples must be sorted and unique") +def sample_identity(value: Any, field: str) -> dict[str, str]: + keys = {"id", "path", "language", "shape"} + if not isinstance(value, dict) or set(value) != keys: + raise ContractError(f"{field} must contain exactly id, path, language, and shape") + if any(not isinstance(value[key], str) or not value[key] for key in keys): + raise ContractError(f"{field} fields must be non-empty strings") + if not value["path"].startswith("samples/") or ".." in Path(value["path"]).parts: + raise ContractError(f"{field}.path must be a safe repository-relative samples/ path") + return {key: value[key] for key in keys} + + +def load_expected(path: Path) -> list[dict[str, str]]: + payload = load_json(path, "sample manifest") + if not isinstance(payload, dict) or not isinstance(payload.get("samples"), list) or not payload["samples"]: + raise ContractError("sample manifest must contain a non-empty samples array") + samples = [sample_identity(value, "manifest sample") for value in payload["samples"]] + ids = [value["id"] for value in samples] + if ids != sorted(set(ids)): + raise ContractError("manifest samples must be sorted and unique by id") return samples -def load_record(path: Path) -> dict[str, Any]: - value = load_json(path, f"result artifact {path.name}") - if not isinstance(value, dict): - raise ContractError(f"result artifact {path.name} must be a JSON object") - if value.get("schema_version") != 1: - raise ContractError(f"result artifact {path.name} must use schema_version 1") - missing = REQUIRED_FIELDS - value.keys() +def load_record(path: Path, expected: dict[str, str]) -> dict[str, Any]: + value = load_json(path, f"result artifact {path}") + if not isinstance(value, dict) or value.get("schema_version") != 1: + raise ContractError("result must be a schema_version 1 object") + missing = REQUIRED - value.keys() if missing: - raise ContractError( - f"result artifact {path.name} is missing fields: {sorted(missing)}" - ) - sample = validate_sample(value["sample"]) - outcome = value["outcome"] - if outcome not in OUTCOMES: - raise ContractError(f"{sample}.outcome is unsupported: {outcome!r}") - if not isinstance(value["stage"], str) or not value["stage"]: - raise ContractError(f"{sample}.stage must be a non-empty string") - if ( - not isinstance(value["duration_seconds"], (int, float)) - or isinstance(value["duration_seconds"], bool) - or value["duration_seconds"] < 0 - ): - raise ContractError(f"{sample}.duration_seconds must be non-negative") - completed_at = parse_timestamp(value["completed_at"], f"{sample}.completed_at") - diagnostic_url = validate_url(value.get("diagnostic_url"), f"{sample}.diagnostic_url") - artifact_url = validate_url(value.get("artifact_url"), f"{sample}.artifact_url") - return { - "sample": sample, - "outcome": outcome, - "stage": value["stage"], - "duration_seconds": value["duration_seconds"], - "completed_at": completed_at, - "diagnostic_url": diagnostic_url, - "artifact_url": artifact_url, - } - - -def collect_records(results_dir: Path, expected: list[str]) -> tuple[list[dict[str, Any]], bool]: + raise ContractError(f"result is missing fields: {sorted(missing)}") + sample = sample_identity(value["sample"], "result sample") + if sample != expected: + raise ContractError(f"sample identity does not match manifest: {sample['id']}") + if value["outcome"] not in OUTCOMES: + raise ContractError(f"unsupported outcome: {value['outcome']!r}") + if not isinstance(value["completed_stage"], str) or not value["completed_stage"]: + raise ContractError("completed_stage must be non-empty") + if not isinstance(value["duration_seconds"], (int, float)) or isinstance(value["duration_seconds"], bool) or value["duration_seconds"] < 0: + raise ContractError("duration_seconds must be non-negative") + timestamp(value["completed_at"], "completed_at") + run = value["run"] + if not isinstance(run, dict) or not RUN_FIELDS <= run.keys(): + raise ContractError(f"run is missing fields: {sorted(RUN_FIELDS - set(run or {}))}") + timestamp(run["started_at"], "run.started_at") + return {**value, "completed_at": timestamp(value["completed_at"], "completed_at")} + + +def collect(results_dir: Path, expected: list[dict[str, str]]) -> tuple[list[dict[str, Any]], bool]: if not results_dir.is_dir(): raise ContractError(f"result artifact directory not found: {results_dir}") + expected_by_id = {value["id"]: value for value in expected} records: dict[str, dict[str, Any]] = {} incomplete = False - for path in sorted(results_dir.glob("*.json")): + for path in sorted(results_dir.glob("*/sample-result.json")): try: - record = load_record(path) + raw = load_json(path, f"result artifact {path}") + sample_id = raw.get("sample", {}).get("id") if isinstance(raw, dict) else None + if sample_id not in expected_by_id: + raise ContractError(f"unexpected sample id: {sample_id}") + if sample_id in records: + raise ContractError(f"duplicate result artifact for {sample_id}") + record = load_record(path, expected_by_id[sample_id]) + if not (path.parent / "diagnostics.log").is_file(): + raise ContractError(f"missing diagnostic: {path.parent / 'diagnostics.log'}") + records[sample_id] = record except ContractError as exc: incomplete = True - records[f""] = { - "sample": f"", - "outcome": "infrastructure_error", - "stage": "reporting", - "duration_seconds": 0, - "completed_at": None, - "diagnostic_url": None, - "artifact_url": None, + records[f"invalid:{path}"] = { + "sample": {"id": path.name, "path": f"", "language": "reporting", "shape": "error"}, + "outcome": "infrastructure/error", "completed_stage": "reporting", + "duration_seconds": 0, "completed_at": None, + "diagnostic_reference": "—", "artifact_reference": path.name, "run": {}, "error": str(exc), } - continue - if record["sample"] in records: - incomplete = True - record["error"] = f"duplicate result artifact for {record['sample']}" - record["outcome"] = "infrastructure_error" - records[record["sample"]] = record - for sample in expected: - if sample not in records: + if sample["id"] not in records: incomplete = True - records[sample] = { - "sample": sample, - "outcome": "infrastructure_error", - "stage": "reporting", - "duration_seconds": 0, - "completed_at": None, - "diagnostic_url": None, - "artifact_url": None, - "error": "expected result artifact is missing", + records[f"missing:{sample['id']}"] = { + "sample": sample, "outcome": "infrastructure/error", + "completed_stage": "reporting", "duration_seconds": 0, + "completed_at": None, "diagnostic_reference": "—", + "artifact_reference": "—", "run": {}, + "error": f"expected result artifact is missing for {sample['id']}", } - return sorted(records.values(), key=lambda record: record["sample"]), incomplete - - -def link(value: str | None) -> str: - if not value: - return "—" - encoded = quote(value, safe=":/?#@!$&'*+,;=%._~-") - return f"[link]({encoded})" + return sorted(records.values(), key=lambda value: value["sample"]["path"]), incomplete def render(records: list[dict[str, Any]], run_url: str | None) -> str: lines = [ - "## Validation report", - "", - "_Run-scoped summary; only attempted samples are listed._", - "", + "## Validation report", "", + "_Run-scoped summary; only attempted samples are listed._", "", "| Sample | Outcome | Completed stage | Duration | Last run (UTC) | Diagnostic/artifact |", "|---|---|---|---:|---|---|", ] for record in records: - completed = ( - record["completed_at"].strftime("%Y-%m-%d %H:%M:%S UTC") - if record["completed_at"] - else "—" - ) - evidence = link(record["diagnostic_url"] or record["artifact_url"]) - sample = f"`{record['sample']}`" - lines.append( - f"| {sample} | {OUTCOMES[record['outcome']]} | {record['stage']} | " - f"{record['duration_seconds']}s | {completed} | {evidence} |" - ) + completed = record["completed_at"].strftime("%Y-%m-%d %H:%M:%S UTC") if record["completed_at"] else "—" + sample = f"`{record['sample']['path']}`" + evidence = f"`{record['diagnostic_reference']}` / `{record['artifact_reference']}`" + lines.append(f"| {sample} | {OUTCOMES[record['outcome']]} | {record['completed_stage']} | {record['duration_seconds']}s | {completed} | {evidence} |") if record.get("error"): - lines.append(f"| `{record['sample']}` | ⚠️ Incomplete | reporting | — | — | {record['error']} |") + lines.append(f"| {sample} | ⚠️ Incomplete | reporting | — | — | {record['error']} |") if run_url: - lines.extend(["", f"Run evidence: {link(run_url)}"]) - lines.extend( - [ - "", - "**Legend:** ✅ passed · ❌ sample failure · ⚠️ infrastructure/error · " - "⏭️ skipped/not-completed", - "", - ] - ) + lines.extend(["", f"Run evidence: {run_url}"]) + lines.extend(["", "**Legend:** ✅ passed · ❌ sample failure · ⚠️ infrastructure/error · ⏭️ skipped/not-completed", ""]) return "\n".join(lines) @@ -217,8 +162,7 @@ def main() -> int: parser.add_argument("--run-url") args = parser.parse_args() try: - expected = load_expected(args.expected_samples) - records, incomplete = collect_records(args.results_dir, expected) + records, incomplete = collect(args.results_dir, load_expected(args.expected_samples)) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(render(records, args.run_url), encoding="utf-8", newline="\n") except (ContractError, OSError) as exc: diff --git a/.github/scripts/test/test-render-validation-report.py b/.github/scripts/test/test-render-validation-report.py index 21f8e9e9d..f24db2381 100644 --- a/.github/scripts/test/test-render-validation-report.py +++ b/.github/scripts/test/test-render-validation-report.py @@ -21,7 +21,17 @@ def setUp(self) -> None: self.results = self.root / "results" self.results.mkdir() self.expected = self.root / "expected.json" - self.expected.write_text(json.dumps(sorted([SAMPLE_A, SAMPLE_B])), encoding="utf-8") + self.expected.write_text( + json.dumps( + { + "samples": [ + {"id": "a", "path": SAMPLE_A, "language": "python", "shape": "quickstart"}, + {"id": "b", "path": SAMPLE_B, "language": "csharp", "shape": "quickstart"}, + ] + } + ), + encoding="utf-8", + ) self.output = self.root / "summary.md" def tearDown(self) -> None: @@ -46,17 +56,35 @@ def run_report(self) -> subprocess.CompletedProcess[str]: ) def write_result(self, sample: str, outcome: str = "passed") -> None: - name = sample.rsplit("/", 1)[-1] + ".json" - (self.results / name).write_text( + sample_id = "a" if sample == SAMPLE_A else "b" + sample_dir = self.results / sample_id + sample_dir.mkdir() + (sample_dir / "diagnostics.log").write_text("diagnostic\n", encoding="utf-8") + (sample_dir / "sample-result.json").write_text( json.dumps( { "schema_version": 1, - "sample": sample, + "sample": { + "id": sample_id, + "path": sample, + "language": "python" if sample_id == "a" else "csharp", + "shape": "quickstart", + }, "outcome": outcome, - "stage": "L3", + "completed_stage": "L3 validation", "duration_seconds": 12.5, + "diagnostic_reference": "diagnostics.log", + "artifact_reference": f"validation-pilot-{sample_id}", "completed_at": "2026-08-10T19:22:33Z", - "diagnostic_url": "https://github.com/example/repo/actions/runs/42", + "run": { + "repository": "example/repo", + "workflow": "validation pilot", + "run_id": "42", + "run_attempt": "1", + "sha": "abc", + "ref": "refs/heads/main", + "started_at": "2026-08-10T19:22:00Z", + }, } ), encoding="utf-8", @@ -64,7 +92,7 @@ def write_result(self, sample: str, outcome: str = "passed") -> None: def test_renders_all_outcomes_and_run_freshness(self) -> None: self.write_result(SAMPLE_A, "passed") - self.write_result(SAMPLE_B, "sample_failure") + self.write_result(SAMPLE_B, "sample failure") completed = self.run_report() self.assertEqual(completed.returncode, 0, completed.stderr) body = self.output.read_text(encoding="utf-8") @@ -83,11 +111,13 @@ def test_missing_expected_artifact_publishes_partial_summary_and_fails(self) -> self.assertIn("⚠️ Infrastructure/error", body) def test_malformed_artifact_publishes_error_row_and_fails(self) -> None: - (self.results / "bad.json").write_text("{", encoding="utf-8") + bad = self.results / "bad" + bad.mkdir() + (bad / "sample-result.json").write_text("{", encoding="utf-8") completed = self.run_report() self.assertEqual(completed.returncode, 1) body = self.output.read_text(encoding="utf-8") - self.assertIn("invalid artifact: bad.json", body) + self.assertIn("invalid artifact: sample-result.json", body) self.assertIn("⚠️ Infrastructure/error", body) diff --git a/.github/workflows/validation-report.yml b/.github/workflows/validation-report.yml index 4b6eb240a..72d71ec4f 100644 --- a/.github/workflows/validation-report.yml +++ b/.github/workflows/validation-report.yml @@ -1,15 +1,14 @@ name: validation report -# Reusable consumer job. The validation producer owns the artifact schema and must -# upload one versioned normalized JSON result per attempted sample plus this sorted -# expected-samples JSON file. This workflow never reads raw logs or language internals. +# Reusable consumer job for validation-pilot v1. The producer owns the schema and +# artifact completeness; this workflow only consumes normalized results and diagnostics. on: workflow_call: inputs: results-artifact: required: true type: string - expected-samples-artifact: + manifest-artifact: required: true type: string @@ -32,7 +31,7 @@ jobs: - name: Download attempted sample list uses: actions/download-artifact@v4 with: - name: ${{ inputs.expected-samples-artifact }} + name: ${{ inputs.manifest-artifact }} path: ${{ runner.temp }}/validation-expected - name: Render run-scoped summary @@ -42,7 +41,7 @@ jobs: set -uo pipefail python .github/scripts/render-validation-report.py \ --results-dir "$RUNNER_TEMP/validation-results" \ - --expected-samples "$RUNNER_TEMP/validation-expected/expected-samples.json" \ + --expected-samples "$RUNNER_TEMP/validation-expected/manifest.json" \ --output "$RUNNER_TEMP/validation-report.md" \ --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" rc=$? From 3d80822b659e62e6dfe6336653b56601c572785b Mon Sep 17 00:00:00 2001 From: brandom Date: Mon, 10 Aug 2026 14:19:07 -0700 Subject: [PATCH 3/3] Use complete validation pilot contract in report Require exact v1 result fields and completeness validation before rendering the run-scoped summary. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/render-validation-report.py | 26 +++++++++++++++---- .../test/test-render-validation-report.py | 1 + .github/workflows/validation-report.yml | 25 ++++++++---------- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/.github/scripts/render-validation-report.py b/.github/scripts/render-validation-report.py index 3eed7ffb5..2e7749e40 100644 --- a/.github/scripts/render-validation-report.py +++ b/.github/scripts/render-validation-report.py @@ -61,7 +61,12 @@ def sample_identity(value: Any, field: str) -> dict[str, str]: def load_expected(path: Path) -> list[dict[str, str]]: payload = load_json(path, "sample manifest") - if not isinstance(payload, dict) or not isinstance(payload.get("samples"), list) or not payload["samples"]: + if ( + not isinstance(payload, dict) + or payload.get("schema_version") != 1 + or not isinstance(payload.get("samples"), list) + or not payload["samples"] + ): raise ContractError("sample manifest must contain a non-empty samples array") samples = [sample_identity(value, "manifest sample") for value in payload["samples"]] ids = [value["id"] for value in samples] @@ -72,7 +77,7 @@ def load_expected(path: Path) -> list[dict[str, str]]: def load_record(path: Path, expected: dict[str, str]) -> dict[str, Any]: value = load_json(path, f"result artifact {path}") - if not isinstance(value, dict) or value.get("schema_version") != 1: + if not isinstance(value, dict) or set(value) != REQUIRED or value.get("schema_version") != 1: raise ContractError("result must be a schema_version 1 object") missing = REQUIRED - value.keys() if missing: @@ -88,9 +93,22 @@ def load_record(path: Path, expected: dict[str, str]) -> dict[str, Any]: raise ContractError("duration_seconds must be non-negative") timestamp(value["completed_at"], "completed_at") run = value["run"] - if not isinstance(run, dict) or not RUN_FIELDS <= run.keys(): + if not isinstance(run, dict) or set(run) != RUN_FIELDS: raise ContractError(f"run is missing fields: {sorted(RUN_FIELDS - set(run or {}))}") timestamp(run["started_at"], "run.started_at") + for field in ("diagnostic_reference", "artifact_reference"): + reference = value[field] + if ( + not isinstance(reference, str) + or not reference + or Path(reference).is_absolute() + or ".." in Path(reference).parts + or len(Path(reference).parts) != 1 + ): + raise ContractError(f"{field} must be a relative filename") + diagnostic = path.parent / value["diagnostic_reference"] + if not diagnostic.is_file(): + raise ContractError(f"missing diagnostic: {diagnostic}") return {**value, "completed_at": timestamp(value["completed_at"], "completed_at")} @@ -109,8 +127,6 @@ def collect(results_dir: Path, expected: list[dict[str, str]]) -> tuple[list[dic if sample_id in records: raise ContractError(f"duplicate result artifact for {sample_id}") record = load_record(path, expected_by_id[sample_id]) - if not (path.parent / "diagnostics.log").is_file(): - raise ContractError(f"missing diagnostic: {path.parent / 'diagnostics.log'}") records[sample_id] = record except ContractError as exc: incomplete = True diff --git a/.github/scripts/test/test-render-validation-report.py b/.github/scripts/test/test-render-validation-report.py index f24db2381..6cdcf6c41 100644 --- a/.github/scripts/test/test-render-validation-report.py +++ b/.github/scripts/test/test-render-validation-report.py @@ -24,6 +24,7 @@ def setUp(self) -> None: self.expected.write_text( json.dumps( { + "schema_version": 1, "samples": [ {"id": "a", "path": SAMPLE_A, "language": "python", "shape": "quickstart"}, {"id": "b", "path": SAMPLE_B, "language": "csharp", "shape": "quickstart"}, diff --git a/.github/workflows/validation-report.yml b/.github/workflows/validation-report.yml index 72d71ec4f..7b2c00554 100644 --- a/.github/workflows/validation-report.yml +++ b/.github/workflows/validation-report.yml @@ -8,9 +8,6 @@ on: results-artifact: required: true type: string - manifest-artifact: - required: true - type: string permissions: contents: read @@ -28,34 +25,34 @@ jobs: name: ${{ inputs.results-artifact }} path: ${{ runner.temp }}/validation-results - - name: Download attempted sample list - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.manifest-artifact }} - path: ${{ runner.temp }}/validation-expected - - - name: Render run-scoped summary + - name: Validate completeness contract id: render if: always() run: | set -uo pipefail + python .github/scripts/validate-validation-pilot-results.py \ + --manifest .github/validation-pilot-matrix.json \ + --artifacts "$RUNNER_TEMP/validation-results" + completeness_rc=$? python .github/scripts/render-validation-report.py \ --results-dir "$RUNNER_TEMP/validation-results" \ - --expected-samples "$RUNNER_TEMP/validation-expected/manifest.json" \ + --expected-samples ".github/validation-pilot-matrix.json" \ --output "$RUNNER_TEMP/validation-report.md" \ --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" - rc=$? + render_rc=$? cat "$RUNNER_TEMP/validation-report.md" >> "$GITHUB_STEP_SUMMARY" - echo "render_rc=$rc" >> "$GITHUB_OUTPUT" + echo "completeness_rc=$completeness_rc" >> "$GITHUB_OUTPUT" + echo "render_rc=$render_rc" >> "$GITHUB_OUTPUT" exit 0 - name: Fail incomplete reporting handoff if: always() env: + COMPLETENESS_RC: ${{ steps.render.outputs.completeness_rc }} RENDER_RC: ${{ steps.render.outputs.render_rc }} run: | set -euo pipefail - if [ "$RENDER_RC" != "0" ]; then + if [ "$COMPLETENESS_RC" != "0" ] || [ "$RENDER_RC" != "0" ]; then echo "::error::Validation reporting handoff was incomplete or malformed." exit 1 fi