ci: report outcomes for manual Full GI runs

.github/scripts/full_gi_summary.py

@@ -0,0 +1,117 @@
+"""Report manual Full GI evidence without changing the integration job result."""
+
+import json
+import os
+import re
+from pathlib import Path
+
+
+def number(value):
+    """Actions outputs are strings; absent or malformed evidence stays unknown."""
+    if isinstance(value, str) and re.fullmatch(r"[0-9]+", value):
+        return int(value)
+    return None
+
+
+def summary(needs):
+    integration = needs.get("integration", {})
+    result = integration.get("result", "")
+    outputs = integration.get("outputs", {})
+    setup = outputs.get("setup_outcome", "")
+    suite = outputs.get("suite_outcome", "")
+    pytest_exit = number(outputs.get("pytest_exit", ""))
+    log_exit = number(outputs.get("log_exit", ""))
+    elapsed = number(outputs.get("elapsed_seconds", ""))
+    upload = outputs.get("upload_outcome", "")
+    artifact_url = outputs.get("artifact_url", "")
+
+    job_status = {
+        "success": "Succeeded",
+        "failure": "Failed",
+        "cancelled": "Cancelled",
+        "skipped": "Skipped",
+    }.get(result, "Incomplete/unknown")
+    setup_status = "Incomplete/unknown — no setup completion evidence"
+    if setup == "success":
+        setup_status = "Completed"
+    elif setup == "skipped" and suite == "skipped" and result == "failure":
+        setup_status = "Failed before the suite; setup completion was not reached"
+    elif setup == "failure":
+        setup_status = "Failed at the setup completion step"
+    elif setup == "cancelled":
+        setup_status = "Cancelled before setup completion"
+
+    if pytest_exit is not None and pytest_exit <= 255:
+        pytest_status = (
+            "Completed successfully (exit 0)"
+            if pytest_exit == 0
+            else f"Failed/errored (exit {pytest_exit})"
+        )
+    else:
+        pytest_status = {
+            "failure": "Failed/interrupted before an exit was recorded",
+            "cancelled": "Cancelled before an exit was recorded",
+            "skipped": "Not run (suite step skipped)",
+        }.get(suite, "Incomplete/unknown — no recorded pytest exit")
+
+    log_status = "Incomplete/unknown — no recorded log capture exit"
+    if log_exit is not None and log_exit <= 255:
+        log_status = (
+            "Captured (tee exit 0)"
+            if log_exit == 0
+            else f"Failed (tee exit {log_exit}); pytest result is separate"
+        )
+
+    artifact_status = "No artifact confirmed"
+    if upload == "success":
+        # Link only the canonical URL emitted by upload-artifact, never arbitrary
+        # Markdown from an unexpected output. Success can also mean no files.
+        if re.fullmatch(
+            r"https://github\.com/[\w.-]+/[\w.-]+/actions/runs/[0-9]+/artifacts/[0-9]+",
+            artifact_url,
+        ):
+            artifact_status = f"[integration-logs]({artifact_url}) uploaded"
+        else:
+            artifact_status = "Upload step succeeded; no artifact URL confirmed"
+    elif upload == "failure":
+        artifact_status = "Upload failed; no artifact confirmed"
+    elif upload == "cancelled":
+        artifact_status = "Upload cancelled; no artifact confirmed"
+    elif upload == "skipped":
+        artifact_status = (
+            "Not requested (successful integration job)"
+            if result == "success"
+            else "Upload skipped; no artifact confirmed"
+        )
+
+    runtime = f"{elapsed}s" if elapsed is not None else "Unknown"
+    rows = (
+        ("Integration job", job_status),
+        ("Setup", setup_status),
+        ("Pytest", pytest_status),
+        ("Log capture", log_status),
+        ("Suite elapsed time", runtime),
+        ("Artifact", artifact_status),
+    )
+    return (
+        "## Full GI outcome\n\n"
+        "| Evidence | Result |\n| --- | --- |\n"
+        + "".join(f"| {label} | {value} |\n" for label, value in rows)
+        + "\nFull GI is manual-only and outside CI OK. A completed pytest result "
+        "does not override job failure or cancellation, and exit 0 alone does "
+        "not establish full training-round lifecycle coverage.\n\n"
+        "Reporting and artifacts are best effort. Cancellation, timeouts or runner "
+        "loss can prevent collection; missing evidence is not a successful suite. "
+        "See the integration job logs for details.\n"
+    )
+
+
+def main():
+    report = summary(json.loads(os.environ["INTEGRATION_NEEDS"]))
+    with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as output:
+        output.write(report)
+    print(report)
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/full-gi.yml

@@ -33,6 +33,14 @@ jobs:
     # A full GI trains 9 clients and runs auditor and aggregator containers.
     # The measured runtime goes to the step summary; tighten this once known.
     timeout-minutes: 90
+    outputs:
+      setup_outcome: ${{ steps.ready.outcome }}
+      suite_outcome: ${{ steps.suite.outcome }}
+      pytest_exit: ${{ steps.suite.outputs.pytest_exit }}
+      log_exit: ${{ steps.suite.outputs.log_exit }}
+      elapsed_seconds: ${{ steps.suite.outputs.elapsed_seconds }}
+      upload_outcome: ${{ steps.upload.outcome }}
+      artifact_url: ${{ steps.upload.outputs.artifact-url }}
     env:
       # Kubo for the harness's offline IPFS daemon; the checksum is the sha256
       # of this release asset.
@@ -129,9 +137,12 @@ jobs:
 
       # Scratch must not exist yet (the harness creates and deletes it); the
       # results dir outlives it and holds the compile, chain and IPFS logs.
-      # A step timeout below the job's, so a hung suite fails this step (and
-      # the upload below still runs) instead of the whole job being cancelled.
+      # Reporting and upload are best effort: setup also consumes the job budget.
+      - name: Integration setup complete
+        id: ready
+        run: echo "Integration setup complete"
       - name: Full-GI integration suite
+        id: suite
         timeout-minutes: 85
         env:
           DIN_TEST_ISOLATED: '1'
@@ -141,20 +152,53 @@ jobs:
         run: |
           mkdir -p "$DIN_TEST_RESULTS_DIR"
           start=$(date +%s)
-          status=0
           set -o pipefail
+          set +e
           pytest tests/dincli/ -v -x -m integration --tb=short 2>&1 \
-            | tee "$DIN_TEST_RESULTS_DIR/pytest.log" || status=$?
+            | tee "$DIN_TEST_RESULTS_DIR/pytest.log"
+          pipeline_status=("${PIPESTATUS[@]}")
+          set -e
+          pytest_exit=${pipeline_status[0]}
+          log_exit=${pipeline_status[1]}
           elapsed=$(( $(date +%s) - start ))
-          echo "Full-GI integration suite: exit ${status} after $((elapsed / 60)) min $((elapsed % 60)) s" \
-            | tee -a "$GITHUB_STEP_SUMMARY"
-          exit "$status"
+          # Evidence-writing failure must also fail this step, without hiding
+          # pytest's original failure when it already returned nonzero.
+          evidence_exit=0
+          {
+            printf 'pytest_exit=%s\n' "$pytest_exit"
+            printf 'log_exit=%s\n' "$log_exit"
+            printf 'elapsed_seconds=%s\n' "$elapsed"
+          } >> "$GITHUB_OUTPUT" || evidence_exit=$?
+          echo "Full-GI: pytest exit $pytest_exit, log capture exit $log_exit, elapsed ${elapsed}s"
+          if [ "$pytest_exit" -ne 0 ]; then exit "$pytest_exit"; fi
+          if [ "$log_exit" -ne 0 ]; then exit "$log_exit"; fi
+          exit "$evidence_exit"
 
       - name: Upload integration logs
-        if: failure()
+        id: upload
+        if: failure() || cancelled()
+        timeout-minutes: 3
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: integration-logs
           path: ${{ runner.temp }}/din-results
           if-no-files-found: warn
           retention-days: 14
+
+  # Independent of the integration runner; missing outputs remain unknown.
+  # Cancellation or runner/platform failure can still prevent this report.
+  integration-summary:
+    name: Full GI summary
+    needs: integration
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        timeout-minutes: 2
+        with:
+          fetch-depth: 1
+      - name: Report Full GI outcome
+        env:
+          INTEGRATION_NEEDS: ${{ toJSON(needs) }}
+        run: python3 .github/scripts/full_gi_summary.py

Developer/CONTRIBUTING.md

@@ -54,7 +54,7 @@ For issue-specific contributor packets, review questions, and curated reading li
   cd foundry && python3 ../.github/scripts/contract_size_gate.py
   ```
   `foundry/anvil.sh` lifts the limit on the local chain, so a local deploy is not a size check. A PR that pushes a contract under the 1,024 B fail line needs a size reduction in the same PR. Don't split a contract without raising it with a maintainer first.
-- **Integration suite.** The `tests/dincli/` full-GI suite (Anvil, an offline Kubo and Docker, in isolated mode; issue #228) is too heavy to run on every push, so it runs only when a maintainer starts the **Full GI** workflow by hand, for example before merging a change to the GI flow. It does not gate `CI OK`. When it fails, the run's `integration-logs` artifact has the pytest output and the compile, chain and IPFS logs. To reproduce it locally, see [CI integration job](../Documentation/technical/testing/dincli-testing-guide.md#ci-integration-job).
+- **Integration suite.** The `tests/dincli/` full-GI suite (Anvil, an offline Kubo and Docker, in isolated mode; issue #228) is too heavy to run on every push, so it runs only when a maintainer starts the **Full GI** workflow by hand, for example before merging a change to the GI flow. It does not gate `CI OK`. Its summary separates setup, pytest, log capture and artifact outcomes. On failure or cancellation, it attempts to upload the available pytest, compile, chain and IPFS logs as `integration-logs`; reporting and upload are best effort. To reproduce it locally, see [CI integration job](../Documentation/technical/testing/dincli-testing-guide.md#ci-integration-job).
 
 ## Code Standards
 

Documentation/technical/testing/dincli-testing-guide.md

@@ -132,10 +132,26 @@ that runs `pytest tests/dincli/ -v -x -m integration --tb=short` with
   its 180 s compile timeout. The solc and Hardhat compiler downloads are
   cached between runs, and the `din-worker` image is built before the suite
   starts rather than inside a Phase 4 command's timeout.
-- **Logs:** when the job fails, the `integration-logs` artifact holds the
-  results directory: `pytest.log`, `hardhat_compile.log`, `forge_build.log`,
-  `chain_node.log` (Anvil), `ipfs_init.log` and `ipfs_daemon.log`. The step
-  summary records the suite's exit status and runtime.
+- **Outcome report:** a small **Full GI summary** job runs after integration,
+  including after failure, and reports the job result, setup completion,
+  recorded pytest exit, log capture exit and suite runtime separately. Missing
+  evidence is unknown, never a successful suite. A completed pytest exit 0
+  does not override a failed or cancelled job, nor prove complete training-round
+  lifecycle coverage. Integration, log capture, upload and reporting failures
+  retain their real failure status; Full GI stays outside `CI OK`.
+- **Logs:** on failure or cancellation, the job attempts a bounded upload of
+  the available results directory as `integration-logs`: `pytest.log`,
+  `hardhat_compile.log`, `forge_build.log`, `chain_node.log` (Anvil),
+  `ipfs_init.log` and `ipfs_daemon.log`. The report links a confirmed artifact
+  separately from the suite result. A successful upload step without an
+  artifact URL does not confirm that files were uploaded. Successful runs do
+  not request an artifact.
+- **Limits:** reporting and uploads are best effort. Setup consumes part of
+  the 90-minute job budget, so the suite's 85-minute timeout does not reserve
+  time for collecting evidence. Cancellation, timeouts or runner loss can
+  prevent outputs, artifacts or even the follow-up report. Use the integration
+  job logs when available; a missing result does not identify a timeout or
+  setup failure by itself.
 
 To reproduce the job locally, use a disposable checkout with no `.env*` file
 other than `.env.example` in the repo root or `foundry/`, Foundry and Kubo
@@ -152,6 +168,7 @@ docker build -f dincli/docker/worker/Dockerfile -t din-worker:dev .
 svm="$HOME/.svm"; [ -d "$svm" ] || svm="${XDG_DATA_HOME:-$HOME/.local/share}/svm"
 export DIN_TEST_ISOLATED=1 DIN_TEST_SVM_DIR="$svm"
 export DIN_TEST_TMPDIR="$(mktemp -d)/scratch" DIN_TEST_RESULTS_DIR="$(mktemp -d)"
+set -o pipefail
 pytest tests/dincli/ -v -x -m integration --tb=short 2>&1 | tee "$DIN_TEST_RESULTS_DIR/pytest.log"
 ```