diff --git a/.github/scripts/full_gi_summary.py b/.github/scripts/full_gi_summary.py new file mode 100644 index 00000000..ed3dc480 --- /dev/null +++ b/.github/scripts/full_gi_summary.py @@ -0,0 +1,117 @@ +"""Report manual Full GI evidence without changing the integration job result.""" + +import json +import os +import re +from pathlib import Path + + +def number(value): + """Actions outputs are strings; absent or malformed evidence stays unknown.""" + if isinstance(value, str) and re.fullmatch(r"[0-9]+", value): + return int(value) + return None + + +def summary(needs): + integration = needs.get("integration", {}) + result = integration.get("result", "") + outputs = integration.get("outputs", {}) + setup = outputs.get("setup_outcome", "") + suite = outputs.get("suite_outcome", "") + pytest_exit = number(outputs.get("pytest_exit", "")) + log_exit = number(outputs.get("log_exit", "")) + elapsed = number(outputs.get("elapsed_seconds", "")) + upload = outputs.get("upload_outcome", "") + artifact_url = outputs.get("artifact_url", "") + + job_status = { + "success": "Succeeded", + "failure": "Failed", + "cancelled": "Cancelled", + "skipped": "Skipped", + }.get(result, "Incomplete/unknown") + setup_status = "Incomplete/unknown — no setup completion evidence" + if setup == "success": + setup_status = "Completed" + elif setup == "skipped" and suite == "skipped" and result == "failure": + setup_status = "Failed before the suite; setup completion was not reached" + elif setup == "failure": + setup_status = "Failed at the setup completion step" + elif setup == "cancelled": + setup_status = "Cancelled before setup completion" + + if pytest_exit is not None and pytest_exit <= 255: + pytest_status = ( + "Completed successfully (exit 0)" + if pytest_exit == 0 + else f"Failed/errored (exit {pytest_exit})" + ) + else: + pytest_status = { + "failure": "Failed/interrupted before an exit was recorded", + "cancelled": "Cancelled before an exit was recorded", + "skipped": "Not run (suite step skipped)", + }.get(suite, "Incomplete/unknown — no recorded pytest exit") + + log_status = "Incomplete/unknown — no recorded log capture exit" + if log_exit is not None and log_exit <= 255: + log_status = ( + "Captured (tee exit 0)" + if log_exit == 0 + else f"Failed (tee exit {log_exit}); pytest result is separate" + ) + + artifact_status = "No artifact confirmed" + if upload == "success": + # Link only the canonical URL emitted by upload-artifact, never arbitrary + # Markdown from an unexpected output. Success can also mean no files. + if re.fullmatch( + r"https://github\.com/[\w.-]+/[\w.-]+/actions/runs/[0-9]+/artifacts/[0-9]+", + artifact_url, + ): + artifact_status = f"[integration-logs]({artifact_url}) uploaded" + else: + artifact_status = "Upload step succeeded; no artifact URL confirmed" + elif upload == "failure": + artifact_status = "Upload failed; no artifact confirmed" + elif upload == "cancelled": + artifact_status = "Upload cancelled; no artifact confirmed" + elif upload == "skipped": + artifact_status = ( + "Not requested (successful integration job)" + if result == "success" + else "Upload skipped; no artifact confirmed" + ) + + runtime = f"{elapsed}s" if elapsed is not None else "Unknown" + rows = ( + ("Integration job", job_status), + ("Setup", setup_status), + ("Pytest", pytest_status), + ("Log capture", log_status), + ("Suite elapsed time", runtime), + ("Artifact", artifact_status), + ) + return ( + "## Full GI outcome\n\n" + "| Evidence | Result |\n| --- | --- |\n" + + "".join(f"| {label} | {value} |\n" for label, value in rows) + + "\nFull GI is manual-only and outside CI OK. A completed pytest result " + "does not override job failure or cancellation, and exit 0 alone does " + "not establish full training-round lifecycle coverage.\n\n" + "Reporting and artifacts are best effort. Cancellation, timeouts or runner " + "loss can prevent collection; missing evidence is not a successful suite. " + "See the integration job logs for details.\n" + ) + + +def main(): + report = summary(json.loads(os.environ["INTEGRATION_NEEDS"])) + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as output: + output.write(report) + print(report) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/full-gi.yml b/.github/workflows/full-gi.yml index 7a185621..493014f9 100644 --- a/.github/workflows/full-gi.yml +++ b/.github/workflows/full-gi.yml @@ -33,6 +33,14 @@ jobs: # A full GI trains 9 clients and runs auditor and aggregator containers. # The measured runtime goes to the step summary; tighten this once known. timeout-minutes: 90 + outputs: + setup_outcome: ${{ steps.ready.outcome }} + suite_outcome: ${{ steps.suite.outcome }} + pytest_exit: ${{ steps.suite.outputs.pytest_exit }} + log_exit: ${{ steps.suite.outputs.log_exit }} + elapsed_seconds: ${{ steps.suite.outputs.elapsed_seconds }} + upload_outcome: ${{ steps.upload.outcome }} + artifact_url: ${{ steps.upload.outputs.artifact-url }} env: # Kubo for the harness's offline IPFS daemon; the checksum is the sha256 # of this release asset. @@ -129,9 +137,12 @@ jobs: # Scratch must not exist yet (the harness creates and deletes it); the # results dir outlives it and holds the compile, chain and IPFS logs. - # A step timeout below the job's, so a hung suite fails this step (and - # the upload below still runs) instead of the whole job being cancelled. + # Reporting and upload are best effort: setup also consumes the job budget. + - name: Integration setup complete + id: ready + run: echo "Integration setup complete" - name: Full-GI integration suite + id: suite timeout-minutes: 85 env: DIN_TEST_ISOLATED: '1' @@ -141,20 +152,53 @@ jobs: run: | mkdir -p "$DIN_TEST_RESULTS_DIR" start=$(date +%s) - status=0 set -o pipefail + set +e pytest tests/dincli/ -v -x -m integration --tb=short 2>&1 \ - | tee "$DIN_TEST_RESULTS_DIR/pytest.log" || status=$? + | tee "$DIN_TEST_RESULTS_DIR/pytest.log" + pipeline_status=("${PIPESTATUS[@]}") + set -e + pytest_exit=${pipeline_status[0]} + log_exit=${pipeline_status[1]} elapsed=$(( $(date +%s) - start )) - echo "Full-GI integration suite: exit ${status} after $((elapsed / 60)) min $((elapsed % 60)) s" \ - | tee -a "$GITHUB_STEP_SUMMARY" - exit "$status" + # Evidence-writing failure must also fail this step, without hiding + # pytest's original failure when it already returned nonzero. + evidence_exit=0 + { + printf 'pytest_exit=%s\n' "$pytest_exit" + printf 'log_exit=%s\n' "$log_exit" + printf 'elapsed_seconds=%s\n' "$elapsed" + } >> "$GITHUB_OUTPUT" || evidence_exit=$? + echo "Full-GI: pytest exit $pytest_exit, log capture exit $log_exit, elapsed ${elapsed}s" + if [ "$pytest_exit" -ne 0 ]; then exit "$pytest_exit"; fi + if [ "$log_exit" -ne 0 ]; then exit "$log_exit"; fi + exit "$evidence_exit" - name: Upload integration logs - if: failure() + id: upload + if: failure() || cancelled() + timeout-minutes: 3 uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: integration-logs path: ${{ runner.temp }}/din-results if-no-files-found: warn retention-days: 14 + + # Independent of the integration runner; missing outputs remain unknown. + # Cancellation or runner/platform failure can still prevent this report. + integration-summary: + name: Full GI summary + needs: integration + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 2 + with: + fetch-depth: 1 + - name: Report Full GI outcome + env: + INTEGRATION_NEEDS: ${{ toJSON(needs) }} + run: python3 .github/scripts/full_gi_summary.py diff --git a/Developer/CONTRIBUTING.md b/Developer/CONTRIBUTING.md index e76e20e6..9a102df6 100644 --- a/Developer/CONTRIBUTING.md +++ b/Developer/CONTRIBUTING.md @@ -54,7 +54,7 @@ For issue-specific contributor packets, review questions, and curated reading li cd foundry && python3 ../.github/scripts/contract_size_gate.py ``` `foundry/anvil.sh` lifts the limit on the local chain, so a local deploy is not a size check. A PR that pushes a contract under the 1,024 B fail line needs a size reduction in the same PR. Don't split a contract without raising it with a maintainer first. -- **Integration suite.** The `tests/dincli/` full-GI suite (Anvil, an offline Kubo and Docker, in isolated mode; issue #228) is too heavy to run on every push, so it runs only when a maintainer starts the **Full GI** workflow by hand, for example before merging a change to the GI flow. It does not gate `CI OK`. When it fails, the run's `integration-logs` artifact has the pytest output and the compile, chain and IPFS logs. To reproduce it locally, see [CI integration job](../Documentation/technical/testing/dincli-testing-guide.md#ci-integration-job). +- **Integration suite.** The `tests/dincli/` full-GI suite (Anvil, an offline Kubo and Docker, in isolated mode; issue #228) is too heavy to run on every push, so it runs only when a maintainer starts the **Full GI** workflow by hand, for example before merging a change to the GI flow. It does not gate `CI OK`. Its summary separates setup, pytest, log capture and artifact outcomes. On failure or cancellation, it attempts to upload the available pytest, compile, chain and IPFS logs as `integration-logs`; reporting and upload are best effort. To reproduce it locally, see [CI integration job](../Documentation/technical/testing/dincli-testing-guide.md#ci-integration-job). ## Code Standards diff --git a/Documentation/technical/testing/dincli-testing-guide.md b/Documentation/technical/testing/dincli-testing-guide.md index e859567e..5036b35a 100644 --- a/Documentation/technical/testing/dincli-testing-guide.md +++ b/Documentation/technical/testing/dincli-testing-guide.md @@ -132,10 +132,26 @@ that runs `pytest tests/dincli/ -v -x -m integration --tb=short` with its 180 s compile timeout. The solc and Hardhat compiler downloads are cached between runs, and the `din-worker` image is built before the suite starts rather than inside a Phase 4 command's timeout. -- **Logs:** when the job fails, the `integration-logs` artifact holds the - results directory: `pytest.log`, `hardhat_compile.log`, `forge_build.log`, - `chain_node.log` (Anvil), `ipfs_init.log` and `ipfs_daemon.log`. The step - summary records the suite's exit status and runtime. +- **Outcome report:** a small **Full GI summary** job runs after integration, + including after failure, and reports the job result, setup completion, + recorded pytest exit, log capture exit and suite runtime separately. Missing + evidence is unknown, never a successful suite. A completed pytest exit 0 + does not override a failed or cancelled job, nor prove complete training-round + lifecycle coverage. Integration, log capture, upload and reporting failures + retain their real failure status; Full GI stays outside `CI OK`. +- **Logs:** on failure or cancellation, the job attempts a bounded upload of + the available results directory as `integration-logs`: `pytest.log`, + `hardhat_compile.log`, `forge_build.log`, `chain_node.log` (Anvil), + `ipfs_init.log` and `ipfs_daemon.log`. The report links a confirmed artifact + separately from the suite result. A successful upload step without an + artifact URL does not confirm that files were uploaded. Successful runs do + not request an artifact. +- **Limits:** reporting and uploads are best effort. Setup consumes part of + the 90-minute job budget, so the suite's 85-minute timeout does not reserve + time for collecting evidence. Cancellation, timeouts or runner loss can + prevent outputs, artifacts or even the follow-up report. Use the integration + job logs when available; a missing result does not identify a timeout or + setup failure by itself. To reproduce the job locally, use a disposable checkout with no `.env*` file other than `.env.example` in the repo root or `foundry/`, Foundry and Kubo @@ -152,6 +168,7 @@ docker build -f dincli/docker/worker/Dockerfile -t din-worker:dev . svm="$HOME/.svm"; [ -d "$svm" ] || svm="${XDG_DATA_HOME:-$HOME/.local/share}/svm" export DIN_TEST_ISOLATED=1 DIN_TEST_SVM_DIR="$svm" export DIN_TEST_TMPDIR="$(mktemp -d)/scratch" DIN_TEST_RESULTS_DIR="$(mktemp -d)" +set -o pipefail pytest tests/dincli/ -v -x -m integration --tb=short 2>&1 | tee "$DIN_TEST_RESULTS_DIR/pytest.log" ``` diff --git a/tests/test_full_gi_summary.py b/tests/test_full_gi_summary.py new file mode 100644 index 00000000..f5a6a978 --- /dev/null +++ b/tests/test_full_gi_summary.py @@ -0,0 +1,281 @@ +"""Offline evidence and failure-propagation checks for manual Full GI reporting.""" + +import json +import os +import re +import subprocess +import sys +import textwrap +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / ".github/scripts/full_gi_summary.py" +SPEC = spec_from_file_location("full_gi_summary", SCRIPT) +reporter = module_from_spec(SPEC) +SPEC.loader.exec_module(reporter) +WORKFLOW = (ROOT / ".github/workflows/full-gi.yml").read_text() + + +def needs(result="success", **outputs): + return {"integration": {"result": result, "outputs": outputs}} + + +def test_completed_suite_with_no_artifact_requested(): + report = reporter.summary( + needs( + setup_outcome="success", + suite_outcome="success", + pytest_exit="0", + log_exit="0", + elapsed_seconds="65", + upload_outcome="skipped", + ) + ) + assert "| Integration job | Succeeded |" in report + assert "| Setup | Completed |" in report + assert "| Pytest | Completed successfully (exit 0) |" in report + assert "| Log capture | Captured (tee exit 0) |" in report + assert "| Suite elapsed time | 65s |" in report + assert "Not requested (successful integration job)" in report + assert "manual-only and outside CI OK" in report + assert ( + "exit 0 alone does not establish full training-round lifecycle coverage" + in report + ) + + +@pytest.mark.parametrize("exit_code", [1, 2, 3, 4, 5, 130, 137]) +def test_nonzero_pytest_is_not_success_even_if_job_reports_success(exit_code): + report = reporter.summary(needs(pytest_exit=str(exit_code))) + assert f"Failed/errored (exit {exit_code})" in report + assert "Completed successfully" not in report + + +def test_setup_failure_before_suite(): + report = reporter.summary( + needs(result="failure", setup_outcome="skipped", suite_outcome="skipped") + ) + assert "| Integration job | Failed |" in report + assert "Failed before the suite" in report + assert "Not run (suite step skipped)" in report + assert "Completed successfully" not in report + + +@pytest.mark.parametrize("result", ["success", "failure", "cancelled", "skipped", ""]) +def test_missing_outputs_never_claim_suite_or_setup_success(result): + report = reporter.summary(needs(result=result)) + assert "no recorded pytest exit" in report + assert "no setup completion evidence" in report + assert "Completed successfully" not in report + assert "No artifact confirmed" in report + assert "| Suite elapsed time | Unknown |" in report + + +def test_missing_integration_is_unknown(): + assert "| Integration job | Incomplete/unknown |" in reporter.summary({}) + + +@pytest.mark.parametrize("value", ["", "garbage", "-1", "256", "0\n", "0.0"]) +def test_malformed_pytest_exit_is_unknown(value): + report = reporter.summary(needs(suite_outcome="success", pytest_exit=value)) + assert "no recorded pytest exit" in report + assert "Completed successfully" not in report + + +@pytest.mark.parametrize( + "outcome,label", [("failure", "Failed/interrupted"), ("cancelled", "Cancelled")] +) +def test_interrupted_suite_without_exit(outcome, label): + report = reporter.summary(needs(result=outcome, suite_outcome=outcome)) + assert f"{label} before an exit was recorded" in report + assert "Completed successfully" not in report + + +@pytest.mark.parametrize( + "result,label", [("failure", "Failed"), ("cancelled", "Cancelled")] +) +def test_completed_pytest_does_not_override_job_result(result, label): + report = reporter.summary(needs(result=result, pytest_exit="0", log_exit="0")) + assert f"| Integration job | {label} |" in report + assert "Completed successfully (exit 0)" in report + + +def test_log_capture_failure_is_separate(): + report = reporter.summary(needs(result="failure", pytest_exit="0", log_exit="7")) + assert "Completed successfully (exit 0)" in report + assert "Failed (tee exit 7); pytest result is separate" in report + + +@pytest.mark.parametrize( + "outcome,label", + [ + ("failure", "Upload failed"), + ("cancelled", "Upload cancelled"), + ("skipped", "Upload skipped"), + ("success", "Upload step succeeded; no artifact URL confirmed"), + ], +) +def test_upload_outcomes_do_not_rewrite_suite(outcome, label): + report = reporter.summary( + needs(result="failure", pytest_exit="1", upload_outcome=outcome) + ) + assert "Failed/errored (exit 1)" in report + assert label in report + assert "](" not in report + + +def test_confirmed_artifact_link(): + url = "https://github.com/InfiniteZeroFoundation/DevNet/actions/runs/123/artifacts/456" + report = reporter.summary(needs(upload_outcome="success", artifact_url=url)) + assert f"[integration-logs]({url}) uploaded" in report + + +def test_unexpected_artifact_output_is_not_rendered_as_markdown(): + report = reporter.summary( + needs(upload_outcome="success", artifact_url="bad)\n| fake | passed |") + ) + assert "no artifact URL confirmed" in report + assert "fake" not in report + + +def job(name, workflow=WORKFLOW): + return re.search( + rf"^ {name}:\n(.*?)(?=^ [a-z][\w-]*:|\Z)", workflow, re.MULTILINE | re.DOTALL + ).group(1) + + +def step_script(step_id): + step = re.search( + rf"^ - name: [^\n]+\n id: {step_id}\n(.*?)(?=^ - |\Z)", + job("integration"), + re.MULTILINE | re.DOTALL, + ).group(1) + return textwrap.dedent(step.split(" run: |\n", 1)[1]) + + +@pytest.mark.parametrize( + "pytest_exit,log_exit,expected", + [(0, 0, 0), (1, 0, 1), (2, 0, 2), (5, 0, 5), (0, 7, 7), (2, 7, 2)], +) +def test_actual_suite_shell_preserves_both_pipeline_exits( + tmp_path, pytest_exit, log_exit, expected +): + fake_pytest = tmp_path / "pytest" + fake_pytest.write_text( + '#!/bin/sh\nprintf "pytest fixture\\n"\nexit "$FAKE_PYTEST_EXIT"\n' + ) + fake_tee = tmp_path / "tee" + fake_tee.write_text('#!/bin/sh\ncat > "$1"\nexit "$FAKE_TEE_EXIT"\n') + fake_pytest.chmod(0o755) + fake_tee.chmod(0o755) + output = tmp_path / "outputs" + result = subprocess.run( + [ + "bash", + "--noprofile", + "--norc", + "-e", + "-o", + "pipefail", + "-c", + step_script("suite"), + ], + env={ + **os.environ, + "PATH": str(tmp_path) + os.pathsep + os.defpath, + "FAKE_PYTEST_EXIT": str(pytest_exit), + "FAKE_TEE_EXIT": str(log_exit), + "DIN_TEST_RESULTS_DIR": str(tmp_path / "results"), + "GITHUB_OUTPUT": str(output), + }, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.returncode == expected, result.stderr + evidence = dict(line.split("=", 1) for line in output.read_text().splitlines()) + assert evidence["pytest_exit"] == str(pytest_exit) + assert evidence["log_exit"] == str(log_exit) + assert int(evidence["elapsed_seconds"]) >= 0 + assert (tmp_path / "results/pytest.log").read_text() == "pytest fixture\n" + + +@pytest.mark.parametrize("pytest_exit,expected", [(0, 1), (2, 2)]) +def test_evidence_write_failure_cannot_hide_pytest_failure( + tmp_path, pytest_exit, expected +): + fake_pytest = tmp_path / "pytest" + fake_pytest.write_text(f"#!/bin/sh\nexit {pytest_exit}\n") + fake_pytest.chmod(0o755) + result = subprocess.run( + ["bash", "-e", "-o", "pipefail", "-c", step_script("suite")], + env={ + **os.environ, + "PATH": str(tmp_path) + os.pathsep + os.defpath, + "DIN_TEST_RESULTS_DIR": str(tmp_path / "results"), + "GITHUB_OUTPUT": str(tmp_path), # A directory cannot receive outputs. + }, + capture_output=True, + timeout=10, + check=False, + ) + assert result.returncode == expected + + +def test_workflow_keeps_manual_policy_and_independent_reporting(): + triggers = WORKFLOW.split("on:\n", 1)[1].split("permissions:", 1)[0] + assert triggers.strip() == "workflow_dispatch:" + assert "continue-on-error" not in WORKFLOW + assert "integration-paths" not in WORKFLOW + integration = job("integration") + assert "needs:" not in integration + for key, expression in { + "setup_outcome": "steps.ready.outcome", + "suite_outcome": "steps.suite.outcome", + "pytest_exit": "steps.suite.outputs.pytest_exit", + "log_exit": "steps.suite.outputs.log_exit", + "elapsed_seconds": "steps.suite.outputs.elapsed_seconds", + "upload_outcome": "steps.upload.outcome", + "artifact_url": "steps.upload.outputs.artifact-url", + }.items(): + assert f"{key}: ${{{{ {expression} }}}}" in integration + assert "if: failure() || cancelled()\n timeout-minutes: 3" in integration + assert "path: ${{ runner.temp }}/din-results" in integration + report = job("integration-summary") + assert "needs: integration" in report + assert "if: always()" in report + assert "timeout-minutes: 5" in report + assert "timeout-minutes: 2" in report + assert "INTEGRATION_NEEDS: ${{ toJSON(needs) }}" in report + assert "run: python3 .github/scripts/full_gi_summary.py" in report + ci = (ROOT / ".github/workflows/ci.yml").read_text() + assert "merge_group:\n types: [checks_requested]" in ci + assert "needs: [solidity, python, docs]" in job("ci-ok", ci) + assert set( + re.findall(r"^ ([a-z][\w-]*):$", ci.split("jobs:\n", 1)[1], re.MULTILINE) + ) == {"solidity", "python", "docs", "ci-ok"} + + +def test_cli_appends_report(tmp_path): + output = tmp_path / "summary" + output.write_text("Existing summary\n") + result = subprocess.run( + [sys.executable, str(SCRIPT)], + env={ + **os.environ, + "INTEGRATION_NEEDS": json.dumps(needs(result="cancelled")), + "GITHUB_STEP_SUMMARY": str(output), + }, + capture_output=True, + text=True, + timeout=10, + check=True, + ) + assert ( + output.read_text() == "Existing summary\n" + result.stdout.rstrip("\n") + "\n" + ) + assert "| Integration job | Cancelled |" in output.read_text()