From 82bbb0f12269cfc6a7754b0b1e5d77a3b8643eee Mon Sep 17 00:00:00 2001 From: Bill Hlavacek Date: Fri, 30 Oct 2026 22:24:40 -0600 Subject: [PATCH 1/3] ci: run the whole suite nightly with every model through the MIR JIT (#1217) BNGSIM_CODEGEN_JIT=mir BNGSIM_CODEGEN_THRESHOLD=1 over python/tests is the run that found #1208, #1212 and #1357. mir.yml runs a hand-picked list of codegen files, so a bug outside that list went unseen until someone ran the whole suite by hand. #1350 made the run a signal: the tests that cannot pass under it skip with declared reasons. This makes it a job. forced-jit.yml: nightly at 07:53 UTC, on workflow_dispatch, and on a pull request that edits it. ubuntu-latest and macos-15, two shards each. The build is python-tests.yml's uv sync --extra dev, with MIR added through SKBUILD_CMAKE_DEFINE as sanitizers.yml adds its flags. A guard fails a build without MIR. The run is strict on skips and replaces crashed workers (--max-worker-restart=500). A failure opens or comments on one tracking issue, as sanitizers.yml does. --- .github/workflows/forced-jit.yml | 170 +++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 .github/workflows/forced-jit.yml diff --git a/.github/workflows/forced-jit.yml b/.github/workflows/forced-jit.yml new file mode 100644 index 00000000..fdf32cf7 --- /dev/null +++ b/.github/workflows/forced-jit.yml @@ -0,0 +1,170 @@ +name: forced-jit + +# Nightly: the whole Python suite with every model compiled through the MIR +# micro-JIT (issue #1217). BNGSIM_CODEGEN_JIT=mir selects the JIT, and +# BNGSIM_CODEGEN_THRESHOLD=1 lowers the auto-codegen threshold from 256 species +# to 1, so a model that would run interpreted is compiled too. +# +# mir.yml runs a hand-picked list of codegen files under the JIT, and a bug +# outside the list is a bug it cannot see. Run by hand, this run found #1208 (a +# call the generated C never declared, which made MIR exit() the pytest worker), +# #1212 (calls the emitter had no C for) and #1357 (a run test the compiled +# march could not take). It is a signal only while it passes, so the tests that +# cannot pass under it say why (python/tests/_codegen_backend.py, #1350): a test +# about cc's .so or its name skips under the JIT, and one that needs a small +# model to run interpreted skips under a lowered threshold, each with a reason +# conftest.py declares. BNGSIM_SKIP_AUDIT=strict holds every other skip to its +# declaration, so a skip nobody decided on fails the run. +# +# Nightly, not per pull request: it is the whole suite, compiled, on two +# platforms. Run it on demand (workflow_dispatch) before merging a change to +# the emitter or the JIT. A pull request that edits this file runs it, so a +# change here is checked before it lands. +# +# Two platforms because the answers differ by platform: #1357's compiled march +# ended at exactly 0 on x86-64 Linux and within 1e-52 of it on arm64 macOS, and +# its run test failed at a different step on each. +# +# The build is python-tests.yml's (uv sync --extra dev: KLU built from source, +# NFsim on), with MIR turned on through SKBUILD_CMAKE_DEFINE, the way +# sanitizers.yml adds its flags. pyproject.toml is untouched, and the suite runs +# as python-tests.yml runs it, in the same two shards. --max-worker-restart: a +# JIT bug can kill a worker (#1208's exit), and xdist stops replacing crashed +# workers after 4 per worker by default, which silently drops the rest of the run. +# +# A failure opens one tracking issue, or comments on it, as sanitizers.yml does. +# +# Scheduled workflows run only from the default branch, and GitHub disables them +# after 60 days without repository activity. + +on: + schedule: + - cron: "53 7 * * *" # 07:53 UTC; off the hour, between sanitizers' 06:37 and nightly-parity's 08:17 + workflow_dispatch: + inputs: + pytest_args: + description: "Extra pytest args (e.g. -x, -k some_pattern, or a single file path)" + required: false + default: "" + pull_request: + paths: + - ".github/workflows/forced-jit.yml" + +permissions: + contents: read + +concurrency: + group: forced-jit-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + suite: + name: Python suite · every model through MIR · ${{ matrix.os }} · ${{ matrix.shard }}/2 + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest # x86_64 Linux + - macos-15 # macOS arm64 + shard: [1, 2] + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + + # One cache per platform, written by shard 1 only, as in python-tests.yml. + # The key ends in -build because the action restores by prefix. + - name: Set up ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: forced-jit-${{ matrix.os }}-build + save: ${{ matrix.shard == 1 }} + max-size: 500M + + - name: Build bngsim (the default configuration, and MIR on) + env: + CMAKE_C_COMPILER_LAUNCHER: ccache + CMAKE_CXX_COMPILER_LAUNCHER: ccache + SKBUILD_CMAKE_DEFINE: BNGSIM_ENABLE_MIR=ON + run: | + set -euxo pipefail + uv sync --extra dev --python 3.12 --reinstall-package bngsim + ccache --show-stats + + # Without MIR compiled in, BNGSIM_CODEGEN_JIT=mir has no JIT to select, + # and a green run would say nothing about it. + - name: Guard against a build without MIR + run: | + set -euxo pipefail + uv run --no-sync python -c "import platform, sys; print(platform.platform()); print(sys.version)" + uv run --no-sync python -c "import bngsim, sys; print('bngsim', bngsim.__version__, 'HAS_MIR', bngsim.HAS_MIR, 'HAS_KLU', bngsim.HAS_KLU); sys.exit(0 if bngsim.HAS_MIR else 'HAS_MIR is False: the MIR backend was not built')" + + - name: Run the suite with every model through the JIT (this shard) + if: ${{ matrix.shard == 1 || !github.event.inputs.pytest_args }} + env: + BNGSIM_CODEGEN_JIT: mir + BNGSIM_CODEGEN_THRESHOLD: "1" + BNGSIM_SKIP_AUDIT: strict + BNGSIM_TEST_DATA: ${{ github.workspace }}/tests/data + BNGSIM_TEST_CACHE_DIR: ${{ runner.temp }}/bngsim-test-cache + BNGSIM_TEST_SHARD: ${{ matrix.shard }}/2 + PYTEST_ARGS: ${{ github.event.inputs.pytest_args }} + run: | + set -o pipefail + if [ -n "$PYTEST_ARGS" ]; then unset BNGSIM_TEST_SHARD; fi + # shellcheck disable=SC2086 # PYTEST_ARGS is words or nothing + uv run --no-sync python -m pytest -p no:cacheprovider python/tests \ + -n auto --dist worksteal --max-worker-restart=500 \ + -q --tb=short -rfE --durations=15 $PYTEST_ARGS \ + 2>&1 | tee forced-jit-${{ matrix.os }}-${{ matrix.shard }}.log + + - name: Upload the pytest log + if: always() + uses: actions/upload-artifact@v4 + with: + name: forced-jit-${{ matrix.os }}-${{ matrix.shard }} + path: forced-jit-${{ matrix.os }}-${{ matrix.shard }}.log + if-no-files-found: ignore + + report: + name: Report + needs: [suite] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - name: Open or update the tracking issue + if: github.event_name != 'pull_request' && needs.suite.result != 'success' + env: + GH_TOKEN: ${{ github.token }} + TITLE: "Nightly forced-JIT suite: failures" + RESULT: ${{ needs.suite.result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + { + echo "Forced-JIT run on \`${GITHUB_SHA::8}\`: $RUN_URL" + echo + echo "- The suite, every model through MIR (\`BNGSIM_CODEGEN_JIT=mir BNGSIM_CODEGEN_THRESHOLD=1\`): **$RESULT**" + echo + echo "Each failed job lists its failures at the end of its log, which is also uploaded. A test that cannot pass under a forced JIT for a reason that is not a bug belongs in \`python/tests/_codegen_backend.py\`, with a reason \`conftest.py\` declares (#1217)." + } > body.md + num=$(gh issue list -R "$GITHUB_REPOSITORY" --state open --search "\"$TITLE\" in:title" \ + --json number,title --jq "map(select(.title == env.TITLE)) | .[0].number // empty") + if [ -n "$num" ]; then + gh issue comment "$num" -R "$GITHUB_REPOSITORY" --body-file body.md + else + gh issue create -R "$GITHUB_REPOSITORY" --title "$TITLE" --body-file body.md + fi + - name: Fail the run on any failure + if: needs.suite.result != 'success' + run: exit 1 From 6a03c27a3ddc95b7aa56103efc9ba94ffc04e353 Mon Sep 17 00:00:00 2001 From: Bill Hlavacek Date: Fri, 30 Oct 2026 22:38:24 -0600 Subject: [PATCH 2/3] tests: check the flip under the MIR JIT too, rather than skip it as no compiler (#1217) test_the_compiled_right_hand_side_flips_where_the_interpreter_does skipped whenever the compiled backend was not cc, with the reason "no C compiler". Under BNGSIM_CODEGEN_JIT=mir the compiler is there and MIR was selected. conftest declares that reason LOCAL_ONLY, so in the forced-JIT job's strict audit the skip failed the run ("!! 2 no C compiler: ..."), with every test passing (17,180 passed, exit 1, macOS arm64). The check holds under MIR: the JIT's run ends where the interpreter's does, to rel 1e-12, on both schedules. It now runs on either compiled backend. Whole suite, BNGSIM_CODEGEN_JIT=mir BNGSIM_CODEGEN_THRESHOLD=1, strict: 17,182 passed, 180 skipped, exit 0. The file by default: 82 passed. --- python/tests/test_stops_on_the_evaluators_flip.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/tests/test_stops_on_the_evaluators_flip.py b/python/tests/test_stops_on_the_evaluators_flip.py index 4fffd165..131d09d3 100644 --- a/python/tests/test_stops_on_the_evaluators_flip.py +++ b/python/tests/test_stops_on_the_evaluators_flip.py @@ -124,14 +124,17 @@ def test_the_compiled_right_hand_side_flips_where_the_interpreter_does(tmp_path, multiply-add, `time() - P*floor(time()/P)` rounds once where the interpreter rounds twice, and flipped a double away at some edges: the compiled run was ten times less accurate at rtol 1e-10. Compiled without - it, the two runs are one.""" + it, the two runs are one. The MIR JIT, which compiles the same source in + process (BNGSIM_CODEGEN_JIT=mir, issue #1217), gives the same run too: + skipping it there skipped the check under the JIT, with a reason that said + the C compiler was missing.""" ends = [] for codegen in (False, True): sim = bngsim.Simulator( _model(tmp_path, _SPELLINGS["remainder"], P, w), method="ode", codegen=codegen ) - if codegen and sim.codegen_backend != "cc": - pytest.skip("no C compiler: the compiled right-hand side is not built with cc") + if codegen and sim.codegen_backend not in ("cc", "mir"): + pytest.skip("no C compiler: the right-hand side is not compiled") r = sim.run(t_span=(0.0, t_end), n_points=11, rtol=1e-10, atol=1e-12) ends.append(float(np.asarray(r.species)[-1, 0])) assert ends[1] == pytest.approx(_on_time(P, w, t_end), rel=5e-9) From 5662153a1aac08f070830442b4e76eeb15ffdb6e Mon Sep 17 00:00:00 2001 From: Bill Hlavacek Date: Fri, 30 Oct 2026 23:16:46 -0600 Subject: [PATCH 3/3] tests: pin BIOMD0000000463's march interpreted; near an unstable state it is the last digits (#1217) The forced-JIT job's first run failed one test on ubuntu-latest: test_a_march_the_step_cannot_stop_returns_the_state_the_residual_test_passed got residual 4.6e17, with a species at -8.5e5. The test pins the march's fallback: a march taken on past the state the residual test passes returns that state. It needs a march that passed the test. Reproduced on x86-64 Linux from 15ba102d's wheel (gcc, no MIR), steady_state(criterion="residual"): interpreted, closed form converged residual 3.9e-10 (the default) interpreted, fd runs away residual 2.6e8 compiled, compiled form runs away residual 4.6e17 (forced codegen) compiled, fd converged residual 1.8e-10 The state has an eigenvalue with real part +0.13, so whether a march passes the residual test before it runs away is decided in the last digits. An interpreted march runs away too, so this is not a compiled-path bug. On macOS arm64 all forced runs passed it. The test now asks for the interpreted march with codegen=False, which is the march the test describes ("main's"), and it keeps running in the forced job. Linux, BNGSIM_CODEGEN_THRESHOLD=1, codegen=False: residual 8.3e-10, min -3.5e-8, run measure 1.2e15, and the four assertions hold. macOS arm64: the file gives 228 passed by default and 227 passed, 3 skipped under BNGSIM_CODEGEN_JIT=mir BNGSIM_CODEGEN_THRESHOLD=1. --- python/tests/test_steady_state_scaled_criterion.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/tests/test_steady_state_scaled_criterion.py b/python/tests/test_steady_state_scaled_criterion.py index 269a3d56..8c127343 100644 --- a/python/tests/test_steady_state_scaled_criterion.py +++ b/python/tests/test_steady_state_scaled_criterion.py @@ -1812,8 +1812,16 @@ def test_a_march_the_step_cannot_stop_returns_the_state_the_residual_test_passed # to a residual of 2e16 with a species at -1.8e5. It returns the state the # residual test passed, main's, where the eigenvalues have a positive real # part (0.13) and a run from it leaves. + # + # The interpreted march, pinned (issue #1217): near an unstable state, + # whether a march passes the residual test before it runs away is in the + # last digits. On x86-64 Linux, interpreted with the closed form it passes + # (3.9e-10), and so does compiled with jacobian="fd" (1.8e-10). Interpreted + # with jacobian="fd" it runs away (2.6e8), and so does compiled with the + # compiled closed form (4.6e17), never passing, so a forced codegen run had + # no passed state to return. path = str(data_dir / "BIOMD0000000463.xml") - ss = bngsim.Simulator(bngsim.Model.from_sbml(path), method="ode").steady_state() + ss = bngsim.Simulator(bngsim.Model.from_sbml(path), method="ode", codegen=False).steady_state() assert not ss.converged assert ss.residual < 1e-9 assert np.min(ss.concentrations) > -1e-6