From 8659583f93b4d4a09c9571327e645dd54abd5471 Mon Sep 17 00:00:00 2001 From: Eric Shenghsiung Liu Date: Fri, 29 May 2026 09:42:44 -0400 Subject: [PATCH 1/6] fix(ci): scope sccache to build step and pin base-anvil-fork SHA Moving RUSTC_WRAPPER and SCCACHE_GHA_ENABLED from the global env block to only the 'Build patched base-anvil binaries' step fixes the 'could not find Cargo.toml' failure: sccache-action and rust-cache were running cargo metadata in the base-std checkout directory (a pure Solidity repo with no Cargo.toml) before the build step cd'd into BASE_ANVIL_DIR. Also pins BASE_ANVIL_REF to a commit SHA instead of a branch name so CI is reproducible and the branch cannot disappear. --- .github/workflows/base-std-fork-tests.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/base-std-fork-tests.yml b/.github/workflows/base-std-fork-tests.yml index c9d3c507..bb9c67db 100644 --- a/.github/workflows/base-std-fork-tests.yml +++ b/.github/workflows/base-std-fork-tests.yml @@ -11,10 +11,9 @@ concurrency: env: BASE_REF: main - BASE_ANVIL_REF: base-anvil-fork + # Pinned to a specific commit so CI is reproducible and the branch can't disappear under us. + BASE_ANVIL_REF: 14a43909528eb7ff4bafab0e5fee9b3b651bcd50 CARGO_TERM_COLOR: always - RUSTC_WRAPPER: "sccache" - SCCACHE_GHA_ENABLED: "true" permissions: contents: read @@ -104,6 +103,8 @@ jobs: shell: bash env: CARGO_PROFILE_RELEASE_LTO: "false" + RUSTC_WRAPPER: "sccache" + SCCACHE_GHA_ENABLED: "true" run: | set -euo pipefail From 1fa829f0c7578072f5f4ed65909c8fa38dafec98 Mon Sep 17 00:00:00 2001 From: Eric Shenghsiung Liu Date: Fri, 29 May 2026 12:39:42 -0400 Subject: [PATCH 2/6] fix(ci): remove rust-cache and fix base-anvil SHA pinning rust-cache ran cargo metadata in the base-std checkout (a Solidity repo with no Cargo.toml), causing the workflow to fail before building anything. Removing it fixes the cargo metadata error; sccache in the build step still provides caching for the actual Rust compilation. Also fixes the base-anvil SHA pinning: git clone --branch does not accept a commit SHA, so CI was failing with 'Remote branch not found'. Solution: clone the named branch and then verify the HEAD SHA matches the pinned value, emitting a warning if the branch has moved. --- .github/workflows/base-std-fork-tests.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/base-std-fork-tests.yml b/.github/workflows/base-std-fork-tests.yml index bb9c67db..a6224189 100644 --- a/.github/workflows/base-std-fork-tests.yml +++ b/.github/workflows/base-std-fork-tests.yml @@ -11,8 +11,9 @@ concurrency: env: BASE_REF: main - # Pinned to a specific commit so CI is reproducible and the branch can't disappear under us. - BASE_ANVIL_REF: 14a43909528eb7ff4bafab0e5fee9b3b651bcd50 + # Branch to clone; SHA below is pinned for reproducibility and verified after clone. + BASE_ANVIL_BRANCH: base-anvil-fork + BASE_ANVIL_SHA: 14a43909528eb7ff4bafab0e5fee9b3b651bcd50 CARGO_TERM_COLOR: always permissions: @@ -70,13 +71,6 @@ jobs: - name: Setup sccache uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - - name: Rust cache - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - with: - cache-on-failure: true - shared-key: "base-std-fork-tests" - cache-targets: "false" - - name: Install Foundry uses: foundry-rs/foundry-toolchain@50d5a8956f2e319df19e6b57539d7e2acb9f8c1e # v1.5.0 with: @@ -93,9 +87,15 @@ jobs: git clone --depth 1 --single-branch --branch "$BASE_REF" \ https://github.com/base/base.git "$workdir/base" - git clone --depth 1 --single-branch --branch "$BASE_ANVIL_REF" \ + + git clone --depth 1 --single-branch --branch "$BASE_ANVIL_BRANCH" \ https://github.com/base/base-anvil.git "$workdir/base-anvil" + actual_sha=$(git -C "$workdir/base-anvil" rev-parse HEAD) + if [ "$actual_sha" != "$BASE_ANVIL_SHA" ]; then + echo "::warning::base-anvil HEAD ($actual_sha) differs from pinned SHA ($BASE_ANVIL_SHA) — update BASE_ANVIL_SHA in the workflow" + fi + echo "BASE_DIR=$workdir/base" >> "$GITHUB_ENV" echo "BASE_ANVIL_DIR=$workdir/base-anvil" >> "$GITHUB_ENV" From bb7a4617b1930b0dc2f1c85f93e00c82b554e6fc Mon Sep 17 00:00:00 2001 From: Eric Shenghsiung Liu Date: Fri, 29 May 2026 14:45:34 -0400 Subject: [PATCH 3/6] fix(ci): make fork tests advisory and add GitHub step summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fork tests now run with continue-on-error: true since base-std is the spec forerunner — divergences mean base/base needs to catch up, not that base-std PRs are wrong. The job still runs and surfaces all failures. A step summary is written to the GitHub Actions Summary tab showing total pass/fail counts and the list of failing tests with a note that failures indicate Rust implementation lag, not base-std bugs. --- .github/workflows/base-std-fork-tests.yml | 34 ++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/base-std-fork-tests.yml b/.github/workflows/base-std-fork-tests.yml index a6224189..7b9a4d68 100644 --- a/.github/workflows/base-std-fork-tests.yml +++ b/.github/workflows/base-std-fork-tests.yml @@ -24,6 +24,9 @@ jobs: name: Base Std Fork Tests runs-on: ubuntu-latest timeout-minutes: 120 + # Advisory: base-std is the spec forerunner. Divergences mean base/base needs to catch up, + # not that base-std is wrong. This job surfaces failures without blocking PR merges. + continue-on-error: true steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 @@ -119,6 +122,7 @@ jobs: "$BASE_ANVIL_DIR/target/release/forge" --version - name: Run base-std fork tests + id: fork_tests shell: bash run: | set -euo pipefail @@ -126,7 +130,35 @@ jobs: ANVIL_BIN="$BASE_ANVIL_DIR/target/release/anvil" \ FORGE_BIN="$BASE_ANVIL_DIR/target/release/forge" \ ANVIL_LOG="$RUNNER_TEMP/base-std-anvil.log" \ - ./script/run-fork-tests.sh + ./script/run-fork-tests.sh 2>&1 | tee "$RUNNER_TEMP/fork-test-output.txt" + echo "fork_tests_exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + continue-on-error: true + + - name: Summarize fork test results + if: always() + shell: bash + run: | + output="$RUNNER_TEMP/fork-test-output.txt" + passed=$(grep -c '\[PASS\]' "$output" 2>/dev/null || echo 0) + failed=$(grep -c '\[FAIL' "$output" 2>/dev/null || echo 0) + + { + echo "## Fork Test Results" + echo "" + if [ "$failed" -eq 0 ]; then + echo "✅ All **${passed}** tests passed — base/base is fully in sync with base-std." + else + echo "⚠️ **${failed}** test(s) failed, **${passed}** passed." + echo "" + echo "These failures indicate divergences where **base/base needs to catch up** to the base-std spec." + echo "base-std PRs are not blocked by this check." + echo "" + echo "### Failing tests" + echo '```' + grep '\[FAIL' "$output" | sed 's/\x1B\[[0-9;]*m//g' || true + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" - name: Upload anvil log if: always() From fca861694bbb146e33995d7235fc83c749eac2cc Mon Sep 17 00:00:00 2001 From: Eric Shenghsiung Liu Date: Fri, 29 May 2026 14:47:48 -0400 Subject: [PATCH 4/6] feat(ci): post fork test results as a PR comment Posts a summary comment directly on the PR after each run showing pass/fail counts and the list of failing tests. Only runs on pull_request events (not merge_group or workflow_dispatch where there is no PR to comment on). Requires pull-requests: write permission. --- .github/workflows/base-std-fork-tests.yml | 32 +++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/base-std-fork-tests.yml b/.github/workflows/base-std-fork-tests.yml index 7b9a4d68..e38d3ab1 100644 --- a/.github/workflows/base-std-fork-tests.yml +++ b/.github/workflows/base-std-fork-tests.yml @@ -18,6 +18,7 @@ env: permissions: contents: read + pull-requests: write jobs: fork-tests: @@ -160,6 +161,37 @@ jobs: fi } >> "$GITHUB_STEP_SUMMARY" + - name: Comment fork test results on PR + if: github.event_name == 'pull_request' && always() + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + output="$RUNNER_TEMP/fork-test-output.txt" + passed=$(grep -c '\[PASS\]' "$output" 2>/dev/null || echo 0) + failed=$(grep -c '\[FAIL' "$output" 2>/dev/null || echo 0) + + if [ "$failed" -eq 0 ]; then + body="### ✅ Fork tests: all ${passed} passed + base/base is fully in sync with the base-std spec." + else + failing=$(grep '\[FAIL' "$output" | sed 's/\x1B\[[0-9;]*m//g' | sed 's/\[FAIL[^]]*\] /- /' | cut -d'(' -f1 | sort -u || true) + body="### ⚠️ Fork tests: ${failed} failed, ${passed} passed + + These failures indicate divergences where **base/base needs to catch up** to the base-std spec. This check is advisory and does not block merging. + +
+ Failing tests + + ${failing} + +
" + fi + + gh pr comment "${{ github.event.pull_request.number }}" \ + --repo "${{ github.repository }}" \ + --body "$body" + - name: Upload anvil log if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 From 8bedd5f0727fe6948f3648290d4cee3fa679b457 Mon Sep 17 00:00:00 2001 From: Eric Shenghsiung Liu Date: Fri, 29 May 2026 14:50:38 -0400 Subject: [PATCH 5/6] feat(ci): find-and-update PR comment with full error details Instead of posting a new comment on every CI run, finds the existing fork-test-results comment (identified by an HTML marker) and updates it in place. Falls back to creating a new comment on first run. Each failing test now shows the test name and its full error inline: - **test_name**: `ErrorType(args)` so divergences are immediately actionable without reading the logs. --- .github/workflows/base-std-fork-tests.yml | 31 ++++++++++++++++++----- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/base-std-fork-tests.yml b/.github/workflows/base-std-fork-tests.yml index e38d3ab1..6ae02c35 100644 --- a/.github/workflows/base-std-fork-tests.yml +++ b/.github/workflows/base-std-fork-tests.yml @@ -170,13 +170,20 @@ jobs: output="$RUNNER_TEMP/fork-test-output.txt" passed=$(grep -c '\[PASS\]' "$output" 2>/dev/null || echo 0) failed=$(grep -c '\[FAIL' "$output" 2>/dev/null || echo 0) + marker="" if [ "$failed" -eq 0 ]; then - body="### ✅ Fork tests: all ${passed} passed + body="${marker} + ### ✅ Fork tests: all ${passed} passed base/base is fully in sync with the base-std spec." else - failing=$(grep '\[FAIL' "$output" | sed 's/\x1B\[[0-9;]*m//g' | sed 's/\[FAIL[^]]*\] /- /' | cut -d'(' -f1 | sort -u || true) - body="### ⚠️ Fork tests: ${failed} failed, ${passed} passed + # Extract test name and full error for each failure, deduplicated + failing=$(grep '\[FAIL' "$output" \ + | sed 's/\x1B\[[0-9;]*m//g' \ + | sed 's/^\[FAIL: \(.*\)\] \(.*\) (runs.*$/- **\2**: `\1`/' \ + | sort -u || true) + body="${marker} + ### ⚠️ Fork tests: ${failed} failed, ${passed} passed These failures indicate divergences where **base/base needs to catch up** to the base-std spec. This check is advisory and does not block merging. @@ -188,9 +195,21 @@ jobs: " fi - gh pr comment "${{ github.event.pull_request.number }}" \ - --repo "${{ github.repository }}" \ - --body "$body" + pr="${{ github.event.pull_request.number }}" + repo="${{ github.repository }}" + + # Find existing comment with our marker and update it, or post a new one + existing_id=$(gh api "repos/${repo}/issues/${pr}/comments" \ + --jq ".[] | select(.body | startswith(\"${marker}\")) | .id" \ + | head -1) + + if [ -n "$existing_id" ]; then + gh api "repos/${repo}/issues/comments/${existing_id}" \ + -X PATCH --field body="$body" > /dev/null + else + gh api "repos/${repo}/issues/${pr}/comments" \ + --field body="$body" > /dev/null + fi - name: Upload anvil log if: always() From 8b6b72793ddacce8b88b469ce64ed3a9d8e80e88 Mon Sep 17 00:00:00 2001 From: Eric Shenghsiung Liu Date: Fri, 29 May 2026 14:55:09 -0400 Subject: [PATCH 6/6] fix(ci): detect build failure before tests run in PR comment When the build or run step fails before any tests execute, the output file is empty or has 0 pass/fail counts. The comment now shows 'Fork tests did not run' with a link to the workflow logs instead of the misleading 'all 0 passed' message. --- .github/workflows/base-std-fork-tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/base-std-fork-tests.yml b/.github/workflows/base-std-fork-tests.yml index 6ae02c35..460e7de4 100644 --- a/.github/workflows/base-std-fork-tests.yml +++ b/.github/workflows/base-std-fork-tests.yml @@ -172,7 +172,11 @@ jobs: failed=$(grep -c '\[FAIL' "$output" 2>/dev/null || echo 0) marker="" - if [ "$failed" -eq 0 ]; then + if [ ! -s "$output" ] || [ "$((passed + failed))" -eq 0 ]; then + body="${marker} + ### ❌ Fork tests did not run + The build or setup step failed before any tests could execute. Check the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details." + elif [ "$failed" -eq 0 ]; then body="${marker} ### ✅ Fork tests: all ${passed} passed base/base is fully in sync with the base-std spec."