From 8ea846cd68a40240f0fcff8f487f9890d1c6a5ca Mon Sep 17 00:00:00 2001 From: Santiagocetran Date: Wed, 28 Oct 2026 16:38:16 +0000 Subject: [PATCH 1/3] CI: improve integration triggers and manual diagnostics --- .github/scripts/integration_ci.py | 181 ++++++++ .github/workflows/ci.yml | 88 ++-- Developer/CONTRIBUTING.md | 2 +- .../technical/testing/dincli-testing-guide.md | 85 +++- tests/test_integration_ci.py | 414 ++++++++++++++++++ 5 files changed, 729 insertions(+), 41 deletions(-) create mode 100644 .github/scripts/integration_ci.py create mode 100644 tests/test_integration_ci.py diff --git a/.github/scripts/integration_ci.py b/.github/scripts/integration_ci.py new file mode 100644 index 00000000..0e845786 --- /dev/null +++ b/.github/scripts/integration_ci.py @@ -0,0 +1,181 @@ +"""Advisory integration selection and reporting; no application imports or APIs.""" + +import argparse +import json +import os +import re +import subprocess +from pathlib import Path + +# Trace/rationale: Documentation/technical/testing/dincli-testing-guide.md. +PREFIXES = ( + "foundry/src/", + "foundry/script/", + "foundry/test/", + "foundry/lib/", + "hardhat/", + "dincli/", + "tests/dincli/", +) +FILES = frozenset( + { + "pyproject.toml", + "MANIFEST.in", + ".github/constraints-ci.txt", + ".gitmodules", + ".dockerignore", + "foundry/anvil.sh", + "foundry/foundry.toml", + "foundry/foundry.lock", + "foundry/remappings.txt", + "foundry/package.json", + "foundry/package-lock.json", + ".github/workflows/ci.yml", + ".github/scripts/integration_ci.py", + } +) +REASONS = { + "manual-enabled": "Manual integration requested.", + "manual-disabled": "Manual integration option was not selected.", + "relevant-paths": "Relevant integration inputs changed.", + "unrelated-paths": "No relevant integration inputs changed.", + "unavailable-base": "Base unavailable; conservatively running integration.", + "unavailable-diff": "Diff unavailable; conservatively running integration.", + "unsupported-event": "Unrecognized event; conservatively running integration.", +} + + +def relevant(path): + return path in FILES or path.startswith(PREFIXES) + + +def git(*args, cwd=None): + return subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + timeout=60, + ).stdout + + +def select(event, before="", manual=False, cwd=None): + """Return an explicit decision/reason; inability to diff never means skip.""" + if event == "workflow_dispatch": + return manual, "manual-enabled" if manual else "manual-disabled" + if event == "pull_request": + base = "HEAD^1" # checkout's merge preview, compared to its base parent + elif event == "push": + if not re.fullmatch(r"[0-9a-fA-F]{40}", before) or set(before) == {"0"}: + return True, "unavailable-base" + base = before + try: + git("fetch", "-q", "--depth=1", "origin", before, cwd=cwd) + except (subprocess.SubprocessError, OSError): + return True, "unavailable-base" + else: + return True, "unsupported-event" + try: + paths = git( + "diff", + "--name-only", + "-z", + "--no-renames", + "--ignore-submodules=none", + base, + "HEAD", + "--", + cwd=cwd, + ) + except (subprocess.SubprocessError, OSError): + return True, "unavailable-diff" + changed = any(relevant(os.fsdecode(path)) for path in paths.split(b"\0") if path) + return changed, "relevant-paths" if changed else "unrelated-paths" + + +def summary(needs): + """Use raw step outcomes, never advisory job success, as suite evidence.""" + detector = needs.get("integration-paths", {}) + integration = needs.get("integration", {}) + decision = detector.get("outputs", {}) + evidence = integration.get("outputs", {}) + run = decision.get("run", "") + reason = decision.get("reason", "") + valid = ( + detector.get("result") == "success" + and run in {"true", "false"} + and reason in REASONS + ) + intentional = ( + valid + and run == "false" + and reason + in { + "manual-disabled", + "unrelated-paths", + } + and integration.get("result") == "skipped" + ) + suite = evidence.get("suite_outcome", "") + status = evidence.get("execution_status", "") + if suite == "success": + outcome = "Passed — integration suite exited successfully" + elif suite == "failure": + outcome = "Failed — integration suite failed" + elif ( + suite == "cancelled" + or status == "cancelled" + or integration.get("result") == "cancelled" + ): + outcome = "Cancelled — no completed suite result" + elif intentional: + outcome = "Intentionally skipped — integration was not requested" + elif status == "failure" and evidence.get("setup_outcome") != "success": + outcome = "Failed — integration setup failed; suite did not run" + else: + outcome = "Incomplete/unknown — no completed suite result" + if not valid: + selection = "Detection failed/incomplete — no valid detector decision; this is not an intentional skip." + else: + selection = REASONS.get(reason, "Detector reason unavailable.") + upload = evidence.get("upload_outcome", "") + logs = ( + "integration-logs artifact uploaded; see the run's artifacts." + if evidence.get("artifact_url") + else "See integration job logs; integration-logs artifact may be absent." + ) + if intentional: + logs = "Integration was not started; no integration logs or artifact expected." + if upload == "failure": + logs += " Artifact upload failed (separate from the suite outcome)." + return ( + "## Integration (advisory)\n\n" + f"**{outcome}.**\n\n{selection}\n\n{logs}\n\n" + "Integration does not gate CI OK. A skipped, failed or incomplete run is not " + "a successful full GI. A passing suite result alone does not establish " + "full-round lifecycle coverage or satisfy the promotion criteria in #270.\n" + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("select", "summary")) + args = parser.parse_args() + if args.command == "select": + run, reason = select( + os.environ.get("EVENT", ""), + os.environ.get("BEFORE", ""), + os.environ.get("MANUAL_INTEGRATION") == "true", + ) + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write(f"run={str(run).lower()}\nreason={reason}\n") + print(REASONS[reason]) + else: + report = summary(json.loads(os.environ["INTEGRATION_NEEDS"])) + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as output: + output.write(report) + print(report) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e39dc5f4..6a80bb19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,13 @@ on: branches: [develop] push: branches: [develop] + # Dispatch is available only after this workflow is also on default main. + workflow_dispatch: + inputs: + run_integration: + description: Run advisory integration for troubleshooting + type: boolean + default: false permissions: contents: read @@ -153,55 +160,50 @@ jobs: # Path filter for the integration job. The workflow itself stays unfiltered # (see ci-ok), so the filter is a job: a change that does not touch the # suite's inputs skips `integration` rather than the whole run. Anything it - # cannot diff (a force push, a missing `before` commit) runs the suite. + # cannot diff (for example, a missing `before` commit) runs the suite. integration-paths: name: Integration paths runs-on: ubuntu-latest timeout-minutes: 5 outputs: run: ${{ steps.diff.outputs.run }} + reason: ${{ steps.diff.outputs.reason }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 2 # a PR's merge commit plus its base parent - - name: Diff against the base + - name: Select integration inputs id: diff env: EVENT: ${{ github.event_name }} BEFORE: ${{ github.event.before }} - run: | - base="" - if [ "$EVENT" = pull_request ]; then - base=HEAD^1 - elif git fetch -q --depth=1 origin "$BEFORE" 2>/dev/null; then - base=$BEFORE - fi - if [ -z "$base" ]; then - echo "no base to diff against; running the suite" - echo "run=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - git diff --name-only "$base" HEAD > "$RUNNER_TEMP/changed.txt" - if grep -qE '^(foundry/(src|script)/|foundry/anvil\.sh$|hardhat/|dincli/|tests/dincli/|\.github/workflows/ci\.yml$)' "$RUNNER_TEMP/changed.txt"; then - echo "run=true" >> "$GITHUB_OUTPUT" - else - echo "run=false" >> "$GITHUB_OUTPUT" - fi - cat "$RUNNER_TEMP/changed.txt" + MANUAL_INTEGRATION: ${{ inputs.run_integration }} + run: python3 .github/scripts/integration_ci.py select # tests/dincli/ full-GI suite on Anvil + offline Kubo + Docker (issue #228). # Non-blocking: ci-ok does not wait for it and continue-on-error keeps a red - # run from failing the workflow. Make it required once it has been stable for - # a couple of weeks: add it to ci-ok's needs and accept `skipped` there. + # run from failing the workflow. Promotion requires separate review and + # successful full-round evidence under #270; a path skip is not that evidence. integration: name: Integration (full GI) needs: integration-paths - if: needs.integration-paths.outputs.run == 'true' + # Fail open on detector failure/missing output, but respect manual opt-out. + if: >- + ${{ !cancelled() && + !(github.event_name == 'workflow_dispatch' && !inputs.run_integration) && + (needs.integration-paths.result != 'success' || + needs.integration-paths.outputs.run != 'false') }} runs-on: ubuntu-latest # A full GI trains 9 clients and runs auditor and aggregator containers. # The measured runtime goes to the step summary; tighten this once known. timeout-minutes: 90 continue-on-error: true + outputs: + suite_outcome: ${{ steps.evidence.outputs.suite_outcome }} + setup_outcome: ${{ steps.evidence.outputs.setup_outcome }} + execution_status: ${{ steps.evidence.outputs.execution_status }} + upload_outcome: ${{ steps.upload.outcome }} + artifact_url: ${{ steps.upload.outputs.artifact-url }} env: # Kubo for the harness's offline IPFS daemon; the checksum is the sha256 # of this release asset. @@ -298,9 +300,12 @@ jobs: # Scratch must not exist yet (the harness creates and deletes it); the # results dir outlives it and holds the compile, chain and IPFS logs. - # A step timeout below the job's, so a hung suite fails this step (and - # the upload below still runs) instead of the whole job being cancelled. + # Reporting/upload are best effort: setup also consumes the job budget. + - name: Integration setup complete + id: ready + run: echo "Integration setup complete" - name: Full-GI integration suite + id: suite timeout-minutes: 85 env: DIN_TEST_ISOLATED: '1' @@ -319,7 +324,21 @@ jobs: | tee -a "$GITHUB_STEP_SUMMARY" exit "$status" + # Capture before upload so an artifact failure cannot change suite status. + - name: Preserve raw integration outcomes + id: evidence + if: always() + env: + SUITE_OUTCOME: ${{ steps.suite.outcome }} + SETUP_OUTCOME: ${{ steps.ready.outcome }} + EXECUTION_STATUS: ${{ job.status }} + run: | + echo "suite_outcome=$SUITE_OUTCOME" >> "$GITHUB_OUTPUT" + echo "setup_outcome=$SETUP_OUTCOME" >> "$GITHUB_OUTPUT" + echo "execution_status=$EXECUTION_STATUS" >> "$GITHUB_OUTPUT" + - name: Upload integration logs + id: upload if: failure() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: @@ -328,6 +347,23 @@ jobs: if-no-files-found: warn retention-days: 14 + # Separate from CI OK: raw step evidence survives advisory job masking. + integration-summary: + name: Integration summary (advisory) + if: always() + needs: [integration-paths, integration] + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + - name: Report integration decision and outcome + env: + INTEGRATION_NEEDS: ${{ toJSON(needs) }} + run: python3 .github/scripts/integration_ci.py summary + # The only required status check. Keeps branch protection to one context and # avoids the path-filter/required-check deadlock. See plan 2.8. ci-ok: diff --git a/Developer/CONTRIBUTING.md b/Developer/CONTRIBUTING.md index b92e17c1..9c56037c 100644 --- a/Developer/CONTRIBUTING.md +++ b/Developer/CONTRIBUTING.md @@ -54,7 +54,7 @@ For issue-specific contributor packets, review questions, and curated reading li cd foundry && python3 ../.github/scripts/contract_size_gate.py ``` `foundry/anvil.sh` lifts the limit on the local chain, so a local deploy is not a size check. A PR that pushes a contract under the 1,024 B fail line needs a size reduction in the same PR. Don't split a contract without raising it with a maintainer first. -- **Integration suite.** A PR that changes `foundry/src/`, `foundry/script/`, `foundry/anvil.sh`, `hardhat/`, `dincli/` or `tests/dincli/` also runs CI's **Integration (full GI)** job: the `tests/dincli/` suite on Anvil, an offline Kubo and Docker, in isolated mode (issue #228). It is non-blocking for now and does not gate `CI OK`. When it fails, the run's `integration-logs` artifact has the pytest output and the compile, chain and IPFS logs. To reproduce it locally, see [CI integration job](../Documentation/technical/testing/dincli-testing-guide.md#ci-integration-job). +- **Integration suite.** Relevant PRs and pushes to `develop` also run CI's advisory **Integration (full GI)** job on Anvil, offline Kubo and Docker in isolated mode (issues #228 and #270). Inputs include CLI/harness code, contract builds and their dependency/configuration files. Integration remains outside the required `CI OK` gate. The run summary distinguishes pass, failure and intentional skip; job logs and any `integration-logs` artifact support troubleshooting. Manual selection is prepared but requires a separate workflow rollout to default branch `main` before GitHub enables dispatch. See [CI integration job](../Documentation/technical/testing/dincli-testing-guide.md#ci-integration-job) for inputs, limitations and reproduction commands. ## Code Standards diff --git a/Documentation/technical/testing/dincli-testing-guide.md b/Documentation/technical/testing/dincli-testing-guide.md index e726f131..44d87467 100644 --- a/Documentation/technical/testing/dincli-testing-guide.md +++ b/Documentation/technical/testing/dincli-testing-guide.md @@ -114,18 +114,64 @@ GitHub/local reproduction command. that runs `pytest tests/dincli/ -v -x -m integration --tb=short` with `DIN_TEST_ISOLATED=1` on a GitHub-hosted runner. -- **When it runs:** on PRs and pushes to `develop` that change - `foundry/src/`, `foundry/script/`, `foundry/anvil.sh`, `hardhat/`, - `dincli/`, `tests/dincli/` or `ci.yml`. (`hardhat/` is included while the - suite still deploys the task contracts from Hardhat artifacts.) The workflow itself has no path filters - because `CI OK` is the single required check, so the small - **Integration paths** job decides by diffing against the base. When it - cannot diff (a force push, for example), the suite runs. -- **Non-blocking:** the job has `continue-on-error: true` and `CI OK` does not - wait for it. It is expected to fail after `test_01` until task_061026_21 - moves the suite to the foundry task contracts and demo wallets. Once it has - been stable for a couple of weeks it can become required: add it to - `ci-ok`'s `needs` and accept `skipped` there. +- **Automatic runs:** PRs and pushes to `develop` run integration when inputs + change. The workflow has no path filters; **Integration paths** compares the + PR merge preview to its first parent, or the push's `before` commit to the + checked-out commit. A missing base, failed fetch or failed diff conservatively + selects integration. Detector failure or missing output also selects it, + except for explicit manual opt-out or workflow cancellation. Missing detector + outputs are never reported as an intentional skip. +- **Manual runs (pending rollout):** `workflow_dispatch` provides the boolean + `run_integration`, default `false`. Once available, choose **Actions → CI → + Run workflow**, select the branch to diagnose, and enable **Run advisory + integration for troubleshooting** to force the suite without a diff. Leaving + it disabled runs only the core checks. GitHub requires the workflow file on + the default branch before dispatch is available. The default branch is + currently `main`, which has no `ci.yml`; merging this change into `develop` + alone does **not** enable manual execution. A separate reviewed rollout to + `main` is required; this change does not alter `main` or repository settings. +- **Non-blocking:** integration and its summary are advisory. `CI OK` still + depends only on Solidity, Python and Docs, and does not wait for integration. + The known old `proxyAdmin` expectation in `test_01_platform.py` remains suite + migration work. A failed or skipped run is not a successful full GI, and a + passing pytest result alone does not establish full-round lifecycle coverage + or satisfy the promotion criteria in issue #270. + +#### Integration inputs + +The filter follows what the current job and harness use: + +| Paths | Why they select integration | +| --- | --- | +| `dincli/`, `tests/dincli/` | CLI, bundled configuration/ABIs/services, worker Dockerfile/entrypoint and integration harness/tests. | +| `pyproject.toml`, `MANIFEST.in`, `.github/constraints-ci.txt` | Editable package/test installation, package file rules and CI dependency constraints. | +| `foundry/src/`, `foundry/script/`, `foundry/test/` | Both CI and the harness run unrestricted `forge build`, including test contracts. | +| `foundry/foundry.toml`, `foundry/foundry.lock`, `foundry/remappings.txt`, `foundry/package.json`, `foundry/package-lock.json` | Compiler/build configuration, library resolution and npm dependencies used by Foundry/FFI. | +| `foundry/lib/`, root `.gitmodules` | Recursive checkout's dependency gitlinks, library sources and submodule configuration. | +| `foundry/anvil.sh`, `hardhat/` | Local chain flags and the Hardhat build/artifacts still used by the suite. | +| `.dockerignore` | Context rules for the worker image build. | +| `.github/workflows/ci.yml`, `.github/scripts/integration_ci.py` | Tool versions, setup, selection and reporting logic. | + +Ordinary documentation and unrelated top-level Python unit tests are excluded. +Although `README.md` supplies package description metadata, prose changes alone +leave integration skipped; the required Python job still installs the package. +`foundry/.gitmodules` is not used by checkout: `foundry` is an ordinary directory, +so the root `.gitmodules` controls its library submodules. + +`cache_model_0/` is **not a direct input to the current integration run**. +`test_03_registration.py` calls `cache-default-artifacts`; the CLI retrieves +manifest, services, ABIs and requirements by CIDs from +`dincli/config/din_info.json`, rather than copying the reference-model tree. +The default test dataset is also retrieved by CID from the CLI. Similarly, +`distribute-mnist` uses CIDs in `dincli/config/cid_services.json`, not the +root `cid_services/` tree. Both bundled configuration files are covered by +`dincli/`. The fresh offline Kubo is currently unseeded; changing triggers does +not repair that suite limitation. Revisit the filter if later fixture work +starts consuming these local trees. Host `.env*` files are not test inputs: +isolated mode rejects them (except `.env.example`, which it does not load). + +#### Execution and results + - **Setup:** Foundry (`FOUNDRY_VERSION`), Node 20, Python 3.12 with CPU torch, Kubo `v0.43.1` (sha256-verified) and the runner's Docker. `forge build` and `npx hardhat compile` run first, in the runner's normal environment, so the @@ -137,8 +183,18 @@ that runs `pytest tests/dincli/ -v -x -m integration --tb=short` with starts rather than inside a Phase 4 command's timeout. - **Logs:** when the job fails, the `integration-logs` artifact holds the results directory: `pytest.log`, `hardhat_compile.log`, `forge_build.log`, - `chain_node.log` (Anvil), `ipfs_init.log` and `ipfs_daemon.log`. The step - summary records the suite's exit status and runtime. + `chain_node.log` (Anvil), `ipfs_init.log` and `ipfs_daemon.log`, when those + files were created. Early setup failures may have only job logs and no + artifact. The pytest step records exit status and elapsed time. +- **Summary:** **Integration summary (advisory)** distinguishes suite passed, + suite failed, intentionally skipped, setup failed, detection failed/incomplete, + and cancelled/incomplete runs. It uses raw step outcomes, not the masked + advisory job result. Artifact-upload failures are reported separately. Only + an explicit successful detector decision to skip is an intentional skip. + Cancellation, runner loss or the job timeout can prevent evidence publication + or the summary itself; missing evidence is unknown, never a pass. Setup time + shares the 90-minute job budget, so the 85-minute suite timeout does not + guarantee time for reporting or upload. To reproduce the job locally, use a disposable checkout with no `.env*` file other than `.env.example` in the repo root or `foundry/`, Foundry and Kubo @@ -155,6 +211,7 @@ docker build -f dincli/docker/worker/Dockerfile -t din-worker:dev . svm="$HOME/.svm"; [ -d "$svm" ] || svm="${XDG_DATA_HOME:-$HOME/.local/share}/svm" export DIN_TEST_ISOLATED=1 DIN_TEST_SVM_DIR="$svm" export DIN_TEST_TMPDIR="$(mktemp -d)/scratch" DIN_TEST_RESULTS_DIR="$(mktemp -d)" +set -o pipefail pytest tests/dincli/ -v -x -m integration --tb=short 2>&1 | tee "$DIN_TEST_RESULTS_DIR/pytest.log" ``` diff --git a/tests/test_integration_ci.py b/tests/test_integration_ci.py new file mode 100644 index 00000000..4db4a323 --- /dev/null +++ b/tests/test_integration_ci.py @@ -0,0 +1,414 @@ +"""Offline contract tests for integration selection, raw reporting and CI wiring.""" + +import json +import os +import re +import subprocess +import sys +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / ".github/scripts/integration_ci.py" +spec = spec_from_file_location("integration_ci", SCRIPT) +ci = module_from_spec(spec) +spec.loader.exec_module(ci) +WORKFLOW = (ROOT / ".github/workflows/ci.yml").read_text() + + +def git(repo, *args): + return subprocess.check_output(["git", *args], cwd=repo).decode().strip() + + +@pytest.fixture +def repo(tmp_path, monkeypatch): + # All remotes below are local disposable directories; no host Git config. + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + monkeypatch.setenv("GIT_CONFIG_GLOBAL", os.devnull) + monkeypatch.setenv("GIT_TERMINAL_PROMPT", "0") + path = tmp_path / "repo" + path.mkdir() + git(path, "init", "-q", "-b", "develop") + git(path, "config", "user.email", "ci@example.invalid") + git(path, "config", "user.name", "CI test") + commit(path, "README.md") + return path + + +def commit(repo, path, content="fixture\n"): + target = repo / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content) + git(repo, "add", "--", path) + git(repo, "commit", "-qm", "fixture") + return git(repo, "rev-parse", "HEAD") + + +def local_origin(repo): + origin = repo.parent / "origin.git" + git(repo, "clone", "-q", "--bare", str(repo), str(origin)) + git(repo, "remote", "add", "origin", str(origin)) + + +@pytest.mark.parametrize( + "path", + [ + "pyproject.toml", + "MANIFEST.in", + ".github/constraints-ci.txt", + ".gitmodules", + ".dockerignore", + "foundry/foundry.toml", + "foundry/foundry.lock", + "foundry/remappings.txt", + "foundry/package.json", + "foundry/package-lock.json", + "foundry/lib/forge-std", + "foundry/lib/openzeppelin-contracts", + "foundry/lib/openzeppelin-contracts-upgradeable", + "foundry/lib/openzeppelin-foundry-upgrades", + "foundry/test/Example.t.sol", + "foundry/src/Example.sol", + "foundry/script/Deploy.s.sol", + "foundry/anvil.sh", + "hardhat/hardhat.config.ts", + "dincli/config/din_info.json", + "dincli/docker/worker/Dockerfile", + "tests/dincli/conftest.py", + ".github/workflows/ci.yml", + ".github/scripts/integration_ci.py", + ], +) +def test_consumed_inputs(path): + assert ci.relevant(path) + + +@pytest.mark.parametrize( + "path", + [ + "README.md", + "Documentation/guide.md", + "tests/test_unrelated.py", + "cache_model_0/manifest.json", + "cid_services/requirements.txt", + "foundry/.gitmodules", + "foundry/README.md", + "pyproject.toml.bak", + "other/dincli/main.py", + ], +) +def test_excluded_inputs(path): + assert not ci.relevant(path) + + +@pytest.mark.parametrize("enabled", [True, False]) +def test_dispatch_does_not_diff_or_fetch(monkeypatch, enabled): + def forbidden(*args, **kwargs): + pytest.fail("manual selection must not execute git") + + monkeypatch.setattr(ci, "git", forbidden) + assert ci.select("workflow_dispatch", manual=enabled) == ( + enabled, + "manual-enabled" if enabled else "manual-disabled", + ) + + +@pytest.mark.parametrize( + "path,expected", + [ + ("Documentation/guide.md", False), + ("pyproject.toml", True), + ("dincli/space tab\tand newline\n.py", True), + (os.fsdecode(b"dincli/nonutf8-\xff.py"), True), + ], +) +def test_real_pr_merge_preview(repo, path, expected): + git(repo, "checkout", "-qb", "feature") + commit(repo, path) + git(repo, "checkout", "-q", "develop") + # Base-only application changes must not be attributed to this PR. + commit(repo, "dincli/base.py") + git(repo, "merge", "--no-ff", "-qm", "preview", "feature") + assert ci.select("pull_request", cwd=repo)[0] is expected + + +def test_push_compares_all_commits(repo): + before = git(repo, "rev-parse", "HEAD") + commit(repo, "pyproject.toml") + commit(repo, "Documentation/guide.md") + local_origin(repo) + assert ci.select("push", before, cwd=repo) == (True, "relevant-paths") + + +def test_docs_only_push(repo): + before = git(repo, "rev-parse", "HEAD") + commit(repo, "Documentation/guide.md") + local_origin(repo) + assert ci.select("push", before, cwd=repo) == (False, "unrelated-paths") + + +def test_force_push_compares_endpoints(repo): + before = commit(repo, "dincli/old.py") + git(repo, "checkout", "--orphan", "replacement") + git(repo, "rm", "-qrf", ".") + commit(repo, "Documentation/guide.md") + local_origin(repo) + assert ci.select("push", before, cwd=repo) == (True, "relevant-paths") + + +@pytest.mark.parametrize("before", ["", "0" * 40, "f" * 40, "--help", "HEAD"]) +def test_unavailable_push_base_runs(repo, before): + local_origin(repo) + assert ci.select("push", before, cwd=repo) == (True, "unavailable-base") + + +def test_missing_pr_parent_runs(repo): + assert ci.select("pull_request", cwd=repo) == (True, "unavailable-diff") + + +@pytest.mark.parametrize( + "error", + [ + subprocess.CalledProcessError(128, "git"), + subprocess.TimeoutExpired("git", 60), + OSError("git unavailable"), + ], +) +def test_failed_diff_runs(monkeypatch, error): + def fail(*args, **kwargs): + raise error + + monkeypatch.setattr(ci, "git", fail) + assert ci.select("pull_request") == (True, "unavailable-diff") + assert ci.select("push", "1" * 40) == (True, "unavailable-base") + + +@pytest.mark.parametrize( + "old,new", + [ + ("dincli/input.py", "Documentation/input.py"), + ("Documentation/input.py", "dincli/input.py"), + ], +) +def test_renames_both_directions(repo, old, new): + commit(repo, old) + (repo / new).parent.mkdir(parents=True, exist_ok=True) + git(repo, "mv", old, new) + git(repo, "commit", "-qm", "rename") + assert ci.select("pull_request", cwd=repo) == (True, "relevant-paths") + + +def test_deletion(repo): + commit(repo, "foundry/foundry.lock") + git(repo, "rm", "foundry/foundry.lock") + git(repo, "commit", "-qm", "delete") + assert ci.select("pull_request", cwd=repo) == (True, "relevant-paths") + + +def test_gitlink_change(repo): + first = git(repo, "rev-parse", "HEAD") + second = commit(repo, "Documentation/guide.md") + for sha in (first, second): + git( + repo, + "update-index", + "--add", + "--cacheinfo", + f"160000,{sha},foundry/lib/forge-std", + ) + git(repo, "commit", "-qm", "gitlink") + assert ci.select("pull_request", cwd=repo) == (True, "relevant-paths") + + +def needs( + run="true", + reason="relevant-paths", + detector="success", + result="success", + **evidence, +): + return { + "integration-paths": { + "result": detector, + "outputs": {"run": run, "reason": reason}, + }, + "integration": {"result": result, "outputs": evidence}, + } + + +@pytest.mark.parametrize( + "outcome,label", [("success", "Passed"), ("failure", "Failed")] +) +def test_raw_suite_outcome_beats_masked_job_success(outcome, label): + report = ci.summary(needs(suite_outcome=outcome, setup_outcome="success")) + assert f"**{label} — integration suite" in report + + +@pytest.mark.parametrize("reason", ["unrelated-paths", "manual-disabled"]) +def test_explicit_skip(reason): + assert "**Intentionally skipped" in ci.summary( + needs(run="false", reason=reason, result="skipped"), + ) + + +@pytest.mark.parametrize( + "data", + [ + {}, + needs(run="", result="skipped"), + needs(run="false", detector="failure", result="skipped"), + needs(run="false", reason="", result="skipped"), + needs(result="skipped"), + needs(run="false", detector="cancelled", result="skipped"), + needs(run="false", result="success"), + ], +) +def test_missing_or_failed_detection_is_never_intentional_skip(data): + report = ci.summary(data) + assert "**Intentionally skipped" not in report + assert "**Passed" not in report + + +def test_detector_failure_is_visible_even_if_fallback_suite_passes(): + report = ci.summary(needs(run="", detector="failure", suite_outcome="success")) + assert "Detection failed/incomplete" in report + assert "suite exited successfully" in report + + +def test_setup_failure_not_suite_failure(): + report = ci.summary( + needs( + execution_status="failure", setup_outcome="skipped", suite_outcome="skipped" + ) + ) + assert "setup failed; suite did not run" in report + assert "artifact may be absent" in report + + +@pytest.mark.parametrize( + "evidence", + [ + {"suite_outcome": "cancelled"}, + {"execution_status": "cancelled"}, + {"result": "cancelled"}, + ], +) +def test_cancellation(evidence): + assert "**Cancelled" in ci.summary(needs(**evidence)) + + +def test_masked_success_without_evidence_is_unknown(): + assert "**Incomplete/unknown" in ci.summary(needs()) + + +def test_upload_failure_cannot_rewrite_suite_result(): + report = ci.summary(needs(suite_outcome="success", upload_outcome="failure")) + assert "**Passed" in report + assert "Artifact upload failed" in report + + +def job(name): + return re.search( + rf"^ {name}:\n(.*?)(?=^ [a-z][\w-]*:|\Z)", WORKFLOW, re.MULTILINE | re.DOTALL + ).group(1) + + +def step(job_text, step_id): + return re.search( + rf" - name: [^\n]+\n id: {step_id}\n(.*?)(?= - |\Z)", + job_text, + re.DOTALL, + ).group(1) + + +def test_workflow_preserves_gate_and_raw_evidence_wiring(): + gate = job("ci-ok") + assert "needs: [solidity, python, docs]" in gate + assert 'if [ "$r" != "success" ]' in gate + assert "if: always()" in gate + assert "continue-on-error" not in gate + assert "paths:" not in WORKFLOW.split("jobs:")[0] + integration = job("integration") + assert "continue-on-error: true" in integration + assert "needs.integration-paths.result != 'success'" in integration + assert "needs.integration-paths.outputs.run != 'false'" in integration + assert "!cancelled()" in integration + assert "!inputs.run_integration" in integration + evidence = step(integration, "evidence") + for raw in ("steps.suite.outcome", "steps.ready.outcome", "job.status"): + assert "${{ " + raw + " }}" in evidence + assert "if: always()" in evidence + assert "steps.evidence.outputs.suite_outcome" in integration + assert "needs: [integration-paths, integration]" in job("integration-summary") + assert "if: always()" in job("integration-summary") + assert "${{ toJSON(needs) }}" in job("integration-summary") + assert "MANUAL_INTEGRATION: ${{ inputs.run_integration }}" in job( + "integration-paths" + ) + + +@pytest.mark.parametrize("exit_code", [0, 1, 2, 5]) +def test_suite_shell_keeps_pytest_exit_status(tmp_path, exit_code): + suite = step(job("integration"), "suite") + script = suite.split(" run: |\n", 1)[1] + script = "\n".join(line[10:] for line in script.splitlines()) + fake = tmp_path / "pytest" + fake.write_text(f"#!/bin/sh\necho 'pytest fixture'\nexit {exit_code}\n") + fake.chmod(0o755) + env = { + **os.environ, + "PATH": str(tmp_path) + os.pathsep + os.defpath, + "DIN_TEST_RESULTS_DIR": str(tmp_path / "results"), + "GITHUB_STEP_SUMMARY": str(tmp_path / "summary"), + } + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", script], + env=env, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.returncode == exit_code + assert f"exit {exit_code} after" in (tmp_path / "summary").read_text() + + +def test_cli_outputs_and_summary(tmp_path): + output = tmp_path / "outputs" + env = { + **os.environ, + "EVENT": "workflow_dispatch", + "MANUAL_INTEGRATION": "false", + "GITHUB_OUTPUT": str(output), + } + subprocess.run([sys.executable, str(SCRIPT), "select"], env=env, check=True) + assert output.read_text() == "run=false\nreason=manual-disabled\n" + report = tmp_path / "summary" + env.update( + INTEGRATION_NEEDS=json.dumps( + needs(run="false", reason="manual-disabled", result="skipped") + ), + GITHUB_STEP_SUMMARY=str(report), + ) + subprocess.run([sys.executable, str(SCRIPT), "summary"], env=env, check=True) + assert "**Intentionally skipped" in report.read_text() + + +def test_push_fetch_success_does_not_hide_diff_failure(monkeypatch): + calls = [] + + def fetch_then_fail(*args, **kwargs): + calls.append(args[0]) + if args[0] == "fetch": + return b"" + raise subprocess.CalledProcessError(128, "git diff") + + monkeypatch.setattr(ci, "git", fetch_then_fail) + assert ci.select("push", "1" * 40) == (True, "unavailable-diff") + assert calls == ["fetch", "diff"] + + +def test_unknown_event_runs_conservatively(): + assert ci.select("unrecognized") == (True, "unsupported-event") From 828426e989a74714baf1f83e8aa8bc8f90d6285c Mon Sep 17 00:00:00 2001 From: Santiagocetran Date: Wed, 28 Oct 2026 16:50:52 +0000 Subject: [PATCH 2/3] CI: include integration parent package initializer --- .github/scripts/integration_ci.py | 1 + Documentation/technical/testing/dincli-testing-guide.md | 2 +- tests/test_integration_ci.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/scripts/integration_ci.py b/.github/scripts/integration_ci.py index 0e845786..1680a043 100644 --- a/.github/scripts/integration_ci.py +++ b/.github/scripts/integration_ci.py @@ -21,6 +21,7 @@ { "pyproject.toml", "MANIFEST.in", + "tests/__init__.py", ".github/constraints-ci.txt", ".gitmodules", ".dockerignore", diff --git a/Documentation/technical/testing/dincli-testing-guide.md b/Documentation/technical/testing/dincli-testing-guide.md index 44d87467..46af330e 100644 --- a/Documentation/technical/testing/dincli-testing-guide.md +++ b/Documentation/technical/testing/dincli-testing-guide.md @@ -143,7 +143,7 @@ The filter follows what the current job and harness use: | Paths | Why they select integration | | --- | --- | -| `dincli/`, `tests/dincli/` | CLI, bundled configuration/ABIs/services, worker Dockerfile/entrypoint and integration harness/tests. | +| `dincli/`, `tests/dincli/`, `tests/__init__.py` | CLI, bundled configuration/ABIs/services, worker Dockerfile/entrypoint and integration harness/tests, including the imported parent package initializer. | | `pyproject.toml`, `MANIFEST.in`, `.github/constraints-ci.txt` | Editable package/test installation, package file rules and CI dependency constraints. | | `foundry/src/`, `foundry/script/`, `foundry/test/` | Both CI and the harness run unrestricted `forge build`, including test contracts. | | `foundry/foundry.toml`, `foundry/foundry.lock`, `foundry/remappings.txt`, `foundry/package.json`, `foundry/package-lock.json` | Compiler/build configuration, library resolution and npm dependencies used by Foundry/FFI. | diff --git a/tests/test_integration_ci.py b/tests/test_integration_ci.py index 4db4a323..a79b118a 100644 --- a/tests/test_integration_ci.py +++ b/tests/test_integration_ci.py @@ -57,6 +57,7 @@ def local_origin(repo): [ "pyproject.toml", "MANIFEST.in", + "tests/__init__.py", ".github/constraints-ci.txt", ".gitmodules", ".dockerignore", @@ -120,6 +121,7 @@ def forbidden(*args, **kwargs): [ ("Documentation/guide.md", False), ("pyproject.toml", True), + ("tests/__init__.py", True), ("dincli/space tab\tand newline\n.py", True), (os.fsdecode(b"dincli/nonutf8-\xff.py"), True), ], From 54f1d4339081898ca766cdd56b61ff197747801f Mon Sep 17 00:00:00 2001 From: Santiagocetran Date: Mon, 2 Nov 2026 14:50:15 +0000 Subject: [PATCH 3/3] ci: report outcomes for manual Full GI runs --- .github/scripts/full_gi_summary.py | 117 ++++++++ .github/workflows/full-gi.yml | 60 +++- Developer/CONTRIBUTING.md | 2 +- .../technical/testing/dincli-testing-guide.md | 25 +- tests/test_full_gi_summary.py | 281 ++++++++++++++++++ 5 files changed, 472 insertions(+), 13 deletions(-) create mode 100644 .github/scripts/full_gi_summary.py create mode 100644 tests/test_full_gi_summary.py diff --git a/.github/scripts/full_gi_summary.py b/.github/scripts/full_gi_summary.py new file mode 100644 index 00000000..ed3dc480 --- /dev/null +++ b/.github/scripts/full_gi_summary.py @@ -0,0 +1,117 @@ +"""Report manual Full GI evidence without changing the integration job result.""" + +import json +import os +import re +from pathlib import Path + + +def number(value): + """Actions outputs are strings; absent or malformed evidence stays unknown.""" + if isinstance(value, str) and re.fullmatch(r"[0-9]+", value): + return int(value) + return None + + +def summary(needs): + integration = needs.get("integration", {}) + result = integration.get("result", "") + outputs = integration.get("outputs", {}) + setup = outputs.get("setup_outcome", "") + suite = outputs.get("suite_outcome", "") + pytest_exit = number(outputs.get("pytest_exit", "")) + log_exit = number(outputs.get("log_exit", "")) + elapsed = number(outputs.get("elapsed_seconds", "")) + upload = outputs.get("upload_outcome", "") + artifact_url = outputs.get("artifact_url", "") + + job_status = { + "success": "Succeeded", + "failure": "Failed", + "cancelled": "Cancelled", + "skipped": "Skipped", + }.get(result, "Incomplete/unknown") + setup_status = "Incomplete/unknown — no setup completion evidence" + if setup == "success": + setup_status = "Completed" + elif setup == "skipped" and suite == "skipped" and result == "failure": + setup_status = "Failed before the suite; setup completion was not reached" + elif setup == "failure": + setup_status = "Failed at the setup completion step" + elif setup == "cancelled": + setup_status = "Cancelled before setup completion" + + if pytest_exit is not None and pytest_exit <= 255: + pytest_status = ( + "Completed successfully (exit 0)" + if pytest_exit == 0 + else f"Failed/errored (exit {pytest_exit})" + ) + else: + pytest_status = { + "failure": "Failed/interrupted before an exit was recorded", + "cancelled": "Cancelled before an exit was recorded", + "skipped": "Not run (suite step skipped)", + }.get(suite, "Incomplete/unknown — no recorded pytest exit") + + log_status = "Incomplete/unknown — no recorded log capture exit" + if log_exit is not None and log_exit <= 255: + log_status = ( + "Captured (tee exit 0)" + if log_exit == 0 + else f"Failed (tee exit {log_exit}); pytest result is separate" + ) + + artifact_status = "No artifact confirmed" + if upload == "success": + # Link only the canonical URL emitted by upload-artifact, never arbitrary + # Markdown from an unexpected output. Success can also mean no files. + if re.fullmatch( + r"https://github\.com/[\w.-]+/[\w.-]+/actions/runs/[0-9]+/artifacts/[0-9]+", + artifact_url, + ): + artifact_status = f"[integration-logs]({artifact_url}) uploaded" + else: + artifact_status = "Upload step succeeded; no artifact URL confirmed" + elif upload == "failure": + artifact_status = "Upload failed; no artifact confirmed" + elif upload == "cancelled": + artifact_status = "Upload cancelled; no artifact confirmed" + elif upload == "skipped": + artifact_status = ( + "Not requested (successful integration job)" + if result == "success" + else "Upload skipped; no artifact confirmed" + ) + + runtime = f"{elapsed}s" if elapsed is not None else "Unknown" + rows = ( + ("Integration job", job_status), + ("Setup", setup_status), + ("Pytest", pytest_status), + ("Log capture", log_status), + ("Suite elapsed time", runtime), + ("Artifact", artifact_status), + ) + return ( + "## Full GI outcome\n\n" + "| Evidence | Result |\n| --- | --- |\n" + + "".join(f"| {label} | {value} |\n" for label, value in rows) + + "\nFull GI is manual-only and outside CI OK. A completed pytest result " + "does not override job failure or cancellation, and exit 0 alone does " + "not establish full training-round lifecycle coverage.\n\n" + "Reporting and artifacts are best effort. Cancellation, timeouts or runner " + "loss can prevent collection; missing evidence is not a successful suite. " + "See the integration job logs for details.\n" + ) + + +def main(): + report = summary(json.loads(os.environ["INTEGRATION_NEEDS"])) + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as output: + output.write(report) + print(report) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/full-gi.yml b/.github/workflows/full-gi.yml index 7a185621..493014f9 100644 --- a/.github/workflows/full-gi.yml +++ b/.github/workflows/full-gi.yml @@ -33,6 +33,14 @@ jobs: # A full GI trains 9 clients and runs auditor and aggregator containers. # The measured runtime goes to the step summary; tighten this once known. timeout-minutes: 90 + outputs: + setup_outcome: ${{ steps.ready.outcome }} + suite_outcome: ${{ steps.suite.outcome }} + pytest_exit: ${{ steps.suite.outputs.pytest_exit }} + log_exit: ${{ steps.suite.outputs.log_exit }} + elapsed_seconds: ${{ steps.suite.outputs.elapsed_seconds }} + upload_outcome: ${{ steps.upload.outcome }} + artifact_url: ${{ steps.upload.outputs.artifact-url }} env: # Kubo for the harness's offline IPFS daemon; the checksum is the sha256 # of this release asset. @@ -129,9 +137,12 @@ jobs: # Scratch must not exist yet (the harness creates and deletes it); the # results dir outlives it and holds the compile, chain and IPFS logs. - # A step timeout below the job's, so a hung suite fails this step (and - # the upload below still runs) instead of the whole job being cancelled. + # Reporting and upload are best effort: setup also consumes the job budget. + - name: Integration setup complete + id: ready + run: echo "Integration setup complete" - name: Full-GI integration suite + id: suite timeout-minutes: 85 env: DIN_TEST_ISOLATED: '1' @@ -141,20 +152,53 @@ jobs: run: | mkdir -p "$DIN_TEST_RESULTS_DIR" start=$(date +%s) - status=0 set -o pipefail + set +e pytest tests/dincli/ -v -x -m integration --tb=short 2>&1 \ - | tee "$DIN_TEST_RESULTS_DIR/pytest.log" || status=$? + | tee "$DIN_TEST_RESULTS_DIR/pytest.log" + pipeline_status=("${PIPESTATUS[@]}") + set -e + pytest_exit=${pipeline_status[0]} + log_exit=${pipeline_status[1]} elapsed=$(( $(date +%s) - start )) - echo "Full-GI integration suite: exit ${status} after $((elapsed / 60)) min $((elapsed % 60)) s" \ - | tee -a "$GITHUB_STEP_SUMMARY" - exit "$status" + # Evidence-writing failure must also fail this step, without hiding + # pytest's original failure when it already returned nonzero. + evidence_exit=0 + { + printf 'pytest_exit=%s\n' "$pytest_exit" + printf 'log_exit=%s\n' "$log_exit" + printf 'elapsed_seconds=%s\n' "$elapsed" + } >> "$GITHUB_OUTPUT" || evidence_exit=$? + echo "Full-GI: pytest exit $pytest_exit, log capture exit $log_exit, elapsed ${elapsed}s" + if [ "$pytest_exit" -ne 0 ]; then exit "$pytest_exit"; fi + if [ "$log_exit" -ne 0 ]; then exit "$log_exit"; fi + exit "$evidence_exit" - name: Upload integration logs - if: failure() + id: upload + if: failure() || cancelled() + timeout-minutes: 3 uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: integration-logs path: ${{ runner.temp }}/din-results if-no-files-found: warn retention-days: 14 + + # Independent of the integration runner; missing outputs remain unknown. + # Cancellation or runner/platform failure can still prevent this report. + integration-summary: + name: Full GI summary + needs: integration + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 2 + with: + fetch-depth: 1 + - name: Report Full GI outcome + env: + INTEGRATION_NEEDS: ${{ toJSON(needs) }} + run: python3 .github/scripts/full_gi_summary.py diff --git a/Developer/CONTRIBUTING.md b/Developer/CONTRIBUTING.md index e76e20e6..9a102df6 100644 --- a/Developer/CONTRIBUTING.md +++ b/Developer/CONTRIBUTING.md @@ -54,7 +54,7 @@ For issue-specific contributor packets, review questions, and curated reading li cd foundry && python3 ../.github/scripts/contract_size_gate.py ``` `foundry/anvil.sh` lifts the limit on the local chain, so a local deploy is not a size check. A PR that pushes a contract under the 1,024 B fail line needs a size reduction in the same PR. Don't split a contract without raising it with a maintainer first. -- **Integration suite.** The `tests/dincli/` full-GI suite (Anvil, an offline Kubo and Docker, in isolated mode; issue #228) is too heavy to run on every push, so it runs only when a maintainer starts the **Full GI** workflow by hand, for example before merging a change to the GI flow. It does not gate `CI OK`. When it fails, the run's `integration-logs` artifact has the pytest output and the compile, chain and IPFS logs. To reproduce it locally, see [CI integration job](../Documentation/technical/testing/dincli-testing-guide.md#ci-integration-job). +- **Integration suite.** The `tests/dincli/` full-GI suite (Anvil, an offline Kubo and Docker, in isolated mode; issue #228) is too heavy to run on every push, so it runs only when a maintainer starts the **Full GI** workflow by hand, for example before merging a change to the GI flow. It does not gate `CI OK`. Its summary separates setup, pytest, log capture and artifact outcomes. On failure or cancellation, it attempts to upload the available pytest, compile, chain and IPFS logs as `integration-logs`; reporting and upload are best effort. To reproduce it locally, see [CI integration job](../Documentation/technical/testing/dincli-testing-guide.md#ci-integration-job). ## Code Standards diff --git a/Documentation/technical/testing/dincli-testing-guide.md b/Documentation/technical/testing/dincli-testing-guide.md index e859567e..5036b35a 100644 --- a/Documentation/technical/testing/dincli-testing-guide.md +++ b/Documentation/technical/testing/dincli-testing-guide.md @@ -132,10 +132,26 @@ that runs `pytest tests/dincli/ -v -x -m integration --tb=short` with its 180 s compile timeout. The solc and Hardhat compiler downloads are cached between runs, and the `din-worker` image is built before the suite starts rather than inside a Phase 4 command's timeout. -- **Logs:** when the job fails, the `integration-logs` artifact holds the - results directory: `pytest.log`, `hardhat_compile.log`, `forge_build.log`, - `chain_node.log` (Anvil), `ipfs_init.log` and `ipfs_daemon.log`. The step - summary records the suite's exit status and runtime. +- **Outcome report:** a small **Full GI summary** job runs after integration, + including after failure, and reports the job result, setup completion, + recorded pytest exit, log capture exit and suite runtime separately. Missing + evidence is unknown, never a successful suite. A completed pytest exit 0 + does not override a failed or cancelled job, nor prove complete training-round + lifecycle coverage. Integration, log capture, upload and reporting failures + retain their real failure status; Full GI stays outside `CI OK`. +- **Logs:** on failure or cancellation, the job attempts a bounded upload of + the available results directory as `integration-logs`: `pytest.log`, + `hardhat_compile.log`, `forge_build.log`, `chain_node.log` (Anvil), + `ipfs_init.log` and `ipfs_daemon.log`. The report links a confirmed artifact + separately from the suite result. A successful upload step without an + artifact URL does not confirm that files were uploaded. Successful runs do + not request an artifact. +- **Limits:** reporting and uploads are best effort. Setup consumes part of + the 90-minute job budget, so the suite's 85-minute timeout does not reserve + time for collecting evidence. Cancellation, timeouts or runner loss can + prevent outputs, artifacts or even the follow-up report. Use the integration + job logs when available; a missing result does not identify a timeout or + setup failure by itself. To reproduce the job locally, use a disposable checkout with no `.env*` file other than `.env.example` in the repo root or `foundry/`, Foundry and Kubo @@ -152,6 +168,7 @@ docker build -f dincli/docker/worker/Dockerfile -t din-worker:dev . svm="$HOME/.svm"; [ -d "$svm" ] || svm="${XDG_DATA_HOME:-$HOME/.local/share}/svm" export DIN_TEST_ISOLATED=1 DIN_TEST_SVM_DIR="$svm" export DIN_TEST_TMPDIR="$(mktemp -d)/scratch" DIN_TEST_RESULTS_DIR="$(mktemp -d)" +set -o pipefail pytest tests/dincli/ -v -x -m integration --tb=short 2>&1 | tee "$DIN_TEST_RESULTS_DIR/pytest.log" ``` diff --git a/tests/test_full_gi_summary.py b/tests/test_full_gi_summary.py new file mode 100644 index 00000000..f5a6a978 --- /dev/null +++ b/tests/test_full_gi_summary.py @@ -0,0 +1,281 @@ +"""Offline evidence and failure-propagation checks for manual Full GI reporting.""" + +import json +import os +import re +import subprocess +import sys +import textwrap +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / ".github/scripts/full_gi_summary.py" +SPEC = spec_from_file_location("full_gi_summary", SCRIPT) +reporter = module_from_spec(SPEC) +SPEC.loader.exec_module(reporter) +WORKFLOW = (ROOT / ".github/workflows/full-gi.yml").read_text() + + +def needs(result="success", **outputs): + return {"integration": {"result": result, "outputs": outputs}} + + +def test_completed_suite_with_no_artifact_requested(): + report = reporter.summary( + needs( + setup_outcome="success", + suite_outcome="success", + pytest_exit="0", + log_exit="0", + elapsed_seconds="65", + upload_outcome="skipped", + ) + ) + assert "| Integration job | Succeeded |" in report + assert "| Setup | Completed |" in report + assert "| Pytest | Completed successfully (exit 0) |" in report + assert "| Log capture | Captured (tee exit 0) |" in report + assert "| Suite elapsed time | 65s |" in report + assert "Not requested (successful integration job)" in report + assert "manual-only and outside CI OK" in report + assert ( + "exit 0 alone does not establish full training-round lifecycle coverage" + in report + ) + + +@pytest.mark.parametrize("exit_code", [1, 2, 3, 4, 5, 130, 137]) +def test_nonzero_pytest_is_not_success_even_if_job_reports_success(exit_code): + report = reporter.summary(needs(pytest_exit=str(exit_code))) + assert f"Failed/errored (exit {exit_code})" in report + assert "Completed successfully" not in report + + +def test_setup_failure_before_suite(): + report = reporter.summary( + needs(result="failure", setup_outcome="skipped", suite_outcome="skipped") + ) + assert "| Integration job | Failed |" in report + assert "Failed before the suite" in report + assert "Not run (suite step skipped)" in report + assert "Completed successfully" not in report + + +@pytest.mark.parametrize("result", ["success", "failure", "cancelled", "skipped", ""]) +def test_missing_outputs_never_claim_suite_or_setup_success(result): + report = reporter.summary(needs(result=result)) + assert "no recorded pytest exit" in report + assert "no setup completion evidence" in report + assert "Completed successfully" not in report + assert "No artifact confirmed" in report + assert "| Suite elapsed time | Unknown |" in report + + +def test_missing_integration_is_unknown(): + assert "| Integration job | Incomplete/unknown |" in reporter.summary({}) + + +@pytest.mark.parametrize("value", ["", "garbage", "-1", "256", "0\n", "0.0"]) +def test_malformed_pytest_exit_is_unknown(value): + report = reporter.summary(needs(suite_outcome="success", pytest_exit=value)) + assert "no recorded pytest exit" in report + assert "Completed successfully" not in report + + +@pytest.mark.parametrize( + "outcome,label", [("failure", "Failed/interrupted"), ("cancelled", "Cancelled")] +) +def test_interrupted_suite_without_exit(outcome, label): + report = reporter.summary(needs(result=outcome, suite_outcome=outcome)) + assert f"{label} before an exit was recorded" in report + assert "Completed successfully" not in report + + +@pytest.mark.parametrize( + "result,label", [("failure", "Failed"), ("cancelled", "Cancelled")] +) +def test_completed_pytest_does_not_override_job_result(result, label): + report = reporter.summary(needs(result=result, pytest_exit="0", log_exit="0")) + assert f"| Integration job | {label} |" in report + assert "Completed successfully (exit 0)" in report + + +def test_log_capture_failure_is_separate(): + report = reporter.summary(needs(result="failure", pytest_exit="0", log_exit="7")) + assert "Completed successfully (exit 0)" in report + assert "Failed (tee exit 7); pytest result is separate" in report + + +@pytest.mark.parametrize( + "outcome,label", + [ + ("failure", "Upload failed"), + ("cancelled", "Upload cancelled"), + ("skipped", "Upload skipped"), + ("success", "Upload step succeeded; no artifact URL confirmed"), + ], +) +def test_upload_outcomes_do_not_rewrite_suite(outcome, label): + report = reporter.summary( + needs(result="failure", pytest_exit="1", upload_outcome=outcome) + ) + assert "Failed/errored (exit 1)" in report + assert label in report + assert "](" not in report + + +def test_confirmed_artifact_link(): + url = "https://github.com/InfiniteZeroFoundation/DevNet/actions/runs/123/artifacts/456" + report = reporter.summary(needs(upload_outcome="success", artifact_url=url)) + assert f"[integration-logs]({url}) uploaded" in report + + +def test_unexpected_artifact_output_is_not_rendered_as_markdown(): + report = reporter.summary( + needs(upload_outcome="success", artifact_url="bad)\n| fake | passed |") + ) + assert "no artifact URL confirmed" in report + assert "fake" not in report + + +def job(name, workflow=WORKFLOW): + return re.search( + rf"^ {name}:\n(.*?)(?=^ [a-z][\w-]*:|\Z)", workflow, re.MULTILINE | re.DOTALL + ).group(1) + + +def step_script(step_id): + step = re.search( + rf"^ - name: [^\n]+\n id: {step_id}\n(.*?)(?=^ - |\Z)", + job("integration"), + re.MULTILINE | re.DOTALL, + ).group(1) + return textwrap.dedent(step.split(" run: |\n", 1)[1]) + + +@pytest.mark.parametrize( + "pytest_exit,log_exit,expected", + [(0, 0, 0), (1, 0, 1), (2, 0, 2), (5, 0, 5), (0, 7, 7), (2, 7, 2)], +) +def test_actual_suite_shell_preserves_both_pipeline_exits( + tmp_path, pytest_exit, log_exit, expected +): + fake_pytest = tmp_path / "pytest" + fake_pytest.write_text( + '#!/bin/sh\nprintf "pytest fixture\\n"\nexit "$FAKE_PYTEST_EXIT"\n' + ) + fake_tee = tmp_path / "tee" + fake_tee.write_text('#!/bin/sh\ncat > "$1"\nexit "$FAKE_TEE_EXIT"\n') + fake_pytest.chmod(0o755) + fake_tee.chmod(0o755) + output = tmp_path / "outputs" + result = subprocess.run( + [ + "bash", + "--noprofile", + "--norc", + "-e", + "-o", + "pipefail", + "-c", + step_script("suite"), + ], + env={ + **os.environ, + "PATH": str(tmp_path) + os.pathsep + os.defpath, + "FAKE_PYTEST_EXIT": str(pytest_exit), + "FAKE_TEE_EXIT": str(log_exit), + "DIN_TEST_RESULTS_DIR": str(tmp_path / "results"), + "GITHUB_OUTPUT": str(output), + }, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.returncode == expected, result.stderr + evidence = dict(line.split("=", 1) for line in output.read_text().splitlines()) + assert evidence["pytest_exit"] == str(pytest_exit) + assert evidence["log_exit"] == str(log_exit) + assert int(evidence["elapsed_seconds"]) >= 0 + assert (tmp_path / "results/pytest.log").read_text() == "pytest fixture\n" + + +@pytest.mark.parametrize("pytest_exit,expected", [(0, 1), (2, 2)]) +def test_evidence_write_failure_cannot_hide_pytest_failure( + tmp_path, pytest_exit, expected +): + fake_pytest = tmp_path / "pytest" + fake_pytest.write_text(f"#!/bin/sh\nexit {pytest_exit}\n") + fake_pytest.chmod(0o755) + result = subprocess.run( + ["bash", "-e", "-o", "pipefail", "-c", step_script("suite")], + env={ + **os.environ, + "PATH": str(tmp_path) + os.pathsep + os.defpath, + "DIN_TEST_RESULTS_DIR": str(tmp_path / "results"), + "GITHUB_OUTPUT": str(tmp_path), # A directory cannot receive outputs. + }, + capture_output=True, + timeout=10, + check=False, + ) + assert result.returncode == expected + + +def test_workflow_keeps_manual_policy_and_independent_reporting(): + triggers = WORKFLOW.split("on:\n", 1)[1].split("permissions:", 1)[0] + assert triggers.strip() == "workflow_dispatch:" + assert "continue-on-error" not in WORKFLOW + assert "integration-paths" not in WORKFLOW + integration = job("integration") + assert "needs:" not in integration + for key, expression in { + "setup_outcome": "steps.ready.outcome", + "suite_outcome": "steps.suite.outcome", + "pytest_exit": "steps.suite.outputs.pytest_exit", + "log_exit": "steps.suite.outputs.log_exit", + "elapsed_seconds": "steps.suite.outputs.elapsed_seconds", + "upload_outcome": "steps.upload.outcome", + "artifact_url": "steps.upload.outputs.artifact-url", + }.items(): + assert f"{key}: ${{{{ {expression} }}}}" in integration + assert "if: failure() || cancelled()\n timeout-minutes: 3" in integration + assert "path: ${{ runner.temp }}/din-results" in integration + report = job("integration-summary") + assert "needs: integration" in report + assert "if: always()" in report + assert "timeout-minutes: 5" in report + assert "timeout-minutes: 2" in report + assert "INTEGRATION_NEEDS: ${{ toJSON(needs) }}" in report + assert "run: python3 .github/scripts/full_gi_summary.py" in report + ci = (ROOT / ".github/workflows/ci.yml").read_text() + assert "merge_group:\n types: [checks_requested]" in ci + assert "needs: [solidity, python, docs]" in job("ci-ok", ci) + assert set( + re.findall(r"^ ([a-z][\w-]*):$", ci.split("jobs:\n", 1)[1], re.MULTILINE) + ) == {"solidity", "python", "docs", "ci-ok"} + + +def test_cli_appends_report(tmp_path): + output = tmp_path / "summary" + output.write_text("Existing summary\n") + result = subprocess.run( + [sys.executable, str(SCRIPT)], + env={ + **os.environ, + "INTEGRATION_NEEDS": json.dumps(needs(result="cancelled")), + "GITHUB_STEP_SUMMARY": str(output), + }, + capture_output=True, + text=True, + timeout=10, + check=True, + ) + assert ( + output.read_text() == "Existing summary\n" + result.stdout.rstrip("\n") + "\n" + ) + assert "| Integration job | Cancelled |" in output.read_text()