Skip to content
E2E + Readiness + Bench

fix(assail): make assail reports deterministic #340

Sign in to view logs

fix(assail): make assail reports deterministic

fix(assail): make assail reports deterministic #340

Workflow file for this run

.github/workflows/e2e.yml at 6a6aca4
# This workflow is managed by gh actions-lock.
# SPDX-License-Identifier: MPL-2.0
# This workflow is managed by gh actions-lock.
# This workflow is managed by gh actions-lock.
# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) <j.d.a.jewell@open.ac.uk>
#
# End-to-end test suite for panic-attack.
# Tests VeriSimDB integration (gracefully skips if gateway unavailable)
# and full scan→detect→report pipeline.
name: E2E + Readiness + Bench
on:
push:
branches: [main, master, develop]
paths:
- 'src/**'
- 'tests/**'
- 'Cargo.toml'
- '.github/workflows/e2e.yml'
pull_request:
branches: [main, master]
paths:
- 'src/**'
- 'tests/**'
- 'Cargo.toml'
workflow_dispatch:
permissions: read-all
concurrency:
group: e2e-${{ github.ref }}
cancel-in-progress: true
jobs:
e2e-verisimdb:
name: E2E — VeriSimDB Integration
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
with:
toolchain: stable
- name: Rust cache
uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Build release binary
run: cargo build --release
- name: Run VeriSimDB E2E
run: bash tests/verisimdb_e2e.sh
e2e-pipeline:
name: E2E — Full Scan Pipeline
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
with:
toolchain: stable
- name: Rust cache
uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Build release binary
run: cargo build --release
- name: E2E — Self-scan produces valid output
run: |
# Scan this repo itself — the tool should analyze its own source
./target/release/panic-attack assail . --output /tmp/self-scan.json
# Verify JSON output is valid
python3 -c "import json; data=json.load(open('/tmp/self-scan.json')); assert 'weak_points' in data or 'summary' in data, 'Missing expected fields'"
echo "PASS: Self-scan produced valid JSON"
- name: E2E — Multi-language detection
run: |
# Create a temp project with multiple language files
mkdir -p /tmp/e2e-multilang/src
cat > /tmp/e2e-multilang/src/main.rs << 'RUST'
fn main() {
let user_input = std::env::args().nth(1).unwrap();
std::process::Command::new("sh").arg("-c").arg(&user_input).output().unwrap();
}
RUST
cat > /tmp/e2e-multilang/src/app.py << 'PYTHON'
import os
user = input()
os.system(user) # command injection
PYTHON
cat > /tmp/e2e-multilang/src/run.sh << 'SHELL'
#!/bin/bash
eval "$1" # command injection
SHELL
# Scan and verify it finds weak points in multiple languages
./target/release/panic-attack assail /tmp/e2e-multilang --output /tmp/multilang-scan.json
echo "PASS: Multi-language scan completed"
- name: E2E — Attestation chain
run: |
set -euo pipefail
# What this replaces: `assail . --output … || true`, then an echo. That
# asserted nothing — no `--attest`, so no chain was built, and `|| true`
# made even a crash a pass. An "attestation chain test" that cannot
# fail is not a test of the attestation chain.
./target/release/panic-attack assail . --attest \
--output /tmp/attested-scan.json --quiet
sidecar=/tmp/attested-scan.attestation.json
test -s "$sidecar" || { echo "::error::no attestation sidecar at $sidecar"; exit 1; }
./target/release/panic-attack attest verify "$sidecar"
python3 - <<'PY'
import json
env = json.load(open("/tmp/attested-scan.attestation.json"))
chain = env["attestation"]
intent, evidence, seal = chain["intent"], chain["evidence"], chain["seal"]
# "Verified" is only meaningful if the chain it verified has content:
# a pre-execution commitment, recorded evidence of what was read, and a
# seal that binds the report hash to the intent.
assert env["envelope_type"] == "trustfile", env["envelope_type"]
assert env["decision_hash"] == seal["report_hash"], "decision hash != seal"
assert len(intent["session_nonce"]) == 64, "intent carries no nonce"
assert evidence["files_read"] > 0, evidence
assert evidence["rolling_content_hash"], evidence
assert seal["intent_hash"], seal
print(f"PASS: attestation chain verified "
f"({evidence['files_read']} files, "
f"{evidence['directories_traversed']} dirs, seal {seal['report_hash'][:12]}…)")
PY
- name: Upload E2E artifacts
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: e2e-scan-results
path: /tmp/*-scan.json
retention-days: 7
# ─── Readiness Grade Tests (CRG: D/C/B tiers) ─────────────────────
readiness:
name: Readiness — Component Grade Verification
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
with:
toolchain: stable
- name: Rust cache
uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Run readiness tests (Grade D/C/B)
run: cargo test --test readiness -- --nocapture
# ─── Benchmarks: Performance Regression Detection ──────────────────
benchmarks:
name: Bench — Scan Performance
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
with:
toolchain: stable
- name: Rust cache
uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- name: Run benchmarks
run: cargo bench 2>&1 | tee /tmp/bench-results.txt
- name: Upload benchmark results
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: benchmark-results
path: /tmp/bench-results.txt
retention-days: 30