Skip to content

bench: measure Python source mutations #89

bench: measure Python source mutations

bench: measure Python source mutations #89

Workflow file for this run

name: CI
on:
push:
branches: [main]
pull_request:
branches: [main]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
RUSTFLAGS: "-D warnings"
jobs:
# README/CHANGELOG/CONTRIBUTING/SECURITY are generated by
# @ktav-lang/polydoc from the root-docs/ unit trees. This job fails when a
# committed artifact drifts from its source — which is what a hand-edit of
# the artifact looks like. The artifacts live partly at the repo root and
# partly under docs/<lang>/ (see the OUT_PATHS table in
# scripts/build-docs.mjs), but all of them are generated: fix the unit
# source and re-run the generator; never patch the artifact.
docs:
name: Docs (generated markdown)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/setup-node@v4
with:
node-version: '24'
cache: 'npm'
- name: Install Node dependencies
run: npm ci
- name: Generated docs are byte-identical to their source units
run: node scripts/build-docs.mjs --check
python-lint:
name: Python lint + typecheck
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: "3.12"
cache: pip
- name: Install lint tooling
# ruff is pinned exactly (kept in sync with the "dev" extra in
# pyproject.toml) so this gate only moves when someone deliberately
# moves it — an unpinned install once went red on a Markdown-
# formatting change nobody in this repo made.
run: python -m pip install ruff==0.16.8 mypy
- name: Ruff check
run: ruff check .
- name: Ruff format check
run: ruff format --check .
- name: Mypy
run: mypy python
rust-lint:
name: Rust fmt + clippy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy
- uses: Swatinem/rust-cache@v2
- name: cargo fmt
run: cargo fmt --check
- name: cargo clippy
run: cargo clippy --all-targets -- -D warnings
test:
name: Test ${{ matrix.os }} / Python ${{ matrix.python }}
runs-on: ${{ matrix.os }}
env:
# tests/conftest.py fails the job if the native extension failed to
# build — a silent regression would otherwise halve the coverage.
KTAV_REQUIRE_NATIVE: "1"
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
python: ["3.9", "3.11", "3.13"]
steps:
- uses: actions/checkout@v6
with:
submodules: recursive
- uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python }}
cache: pip
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Create venv
run: python -m venv .venv
- name: Export venv to PATH and VIRTUAL_ENV
shell: python
run: |
import os, sys
venv = os.path.abspath(".venv")
bin_dir = os.path.join(venv, "Scripts" if sys.platform == "win32" else "bin")
with open(os.environ["GITHUB_PATH"], "a") as f:
f.write(bin_dir + "\n")
with open(os.environ["GITHUB_ENV"], "a") as f:
f.write(f"VIRTUAL_ENV={venv}\n")
- name: Install test tooling
run: python -m pip install --upgrade pip maturin pytest pytest-cov "tomli>=2.5.0; python_version < '3.11'"
# Lock + provenance check before the native build compiles anything.
# The core is an IMMUTABLE git pin (Cargo.toml rev 8b6b45ff...);
# the verification step below FAILS CLOSED on rev/version drift —
# it never skips unconditionally.
- name: Generate Cargo.lock (no --locked)
run: cargo generate-lockfile
- name: Verify ktav core pin (rev/version, fail closed)
shell: python
run: |
import re, sys
from pathlib import Path
PIN = "8b6b45ffbd9e50e1588e8ab6b1ae5d422bbe8910"
toml = Path("Cargo.toml").read_text(encoding="utf-8")
m = re.search(r'rev\s*=\s*"([0-9a-f]{40})"', toml)
if m is None:
sys.exit("Cargo.toml: ktav dependency has no git rev pin")
if m.group(1) != PIN:
sys.exit(f"Cargo.toml rev {m.group(1)} != immutable pin {PIN}")
lock = Path("Cargo.lock").read_text(encoding="utf-8")
packages = lock.split("[[package]]")
core = [p for p in packages if re.search(r'^name = "ktav"$', p, re.M)]
if len(core) != 1:
sys.exit("Cargo.lock: expected exactly one ktav package")
entry = core[0]
sm = re.search(r'^source = "git\+https://github\.com/ktav-lang/rust\?rev=' + PIN + r'#([0-9a-f]{40})"$', entry, re.M)
if sm is None:
sys.exit("Cargo.lock: ktav is not locked to the pinned git source")
if sm.group(1) != PIN:
sys.exit(f"Cargo.lock rev {sm.group(1)} != immutable pin {PIN}")
vm = re.search(r'version = "([^"]+)"', entry)
if vm is None:
sys.exit("Cargo.lock: ktav entry has no version")
if vm.group(1) != "0.9.0":
sys.exit(f"Cargo.lock: expected ktav core 0.9.0, got {vm.group(1)}")
print(f"ktav core pinned: rev {PIN}, version {vm.group(1)}")
- name: Build extension (release)
run: maturin develop --release --locked
- name: Pytest (both backends; conftest enforces the native scenario)
run: pytest -v
# Pure-Python backend, no native extension, on the support matrix
# edges (oldest and newest CPython, both CI-relevant OSes). Installs
# ONLY pytest — no maturin, no Rust toolchain — and runs the
# backend-parametrized conformance runner plus the tests/pure and
# tests/edit packages
# with ktav._core guaranteed absent (tests/conftest.py fails the job if
# it is not). The native-only unit suites (tests/parse, tests/serialize,
# tests/roundtrip, tests/api, tests/spec) call the native-default
# top-level API and are exercised by the `test` job instead.
#
# The packaged scenario then builds a REAL native-free pure wheel from
# packaging/pure/ (setuptools backend — no Rust, no maturin anywhere in
# this job), asserts its exact filename/version, installs it into a
# FRESH venv, and re-runs the full five-category corpus through ktav.pure
# from the INSTALLED package — where ktav._core is genuinely ABSENT (the
# pure wheel cannot contain it). Missing corpus, wrong § 8.5 counts or a
# non-functional pure backend fail the job; nothing skips. The pure wheel
# is built FIRST and fails fast; no native-wheel step here could mask it
# (there is none).
pure-mode:
name: Pure (no native) ${{ matrix.os }} / Python ${{ matrix.python }}
runs-on: ${{ matrix.os }}
env:
KTAV_REQUIRE_PURE: "1"
PYTHONPATH: python
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest]
python: ["3.9", "3.13"]
steps:
- uses: actions/checkout@v6
with:
submodules: recursive
- uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python }}
- name: Cross-check hardcoded pure-wheel version against Cargo.toml
shell: python
run: |
import re, sys
cargo = open("Cargo.toml", encoding="utf-8").read()
pure = open("packaging/pure/pyproject.toml", encoding="utf-8").read()
cv = re.search(r'^version\s*=\s*"([^"]+)"', cargo, re.M).group(1)
pv = re.search(r'^version\s*=\s*"([^"]+)"', pure, re.M).group(1)
if cv != pv or pv != "0.8.1":
sys.exit(
f"version mismatch: Cargo.toml={cv!r} "
f"packaging/pure={pv!r} expected 0.8.1"
)
- name: Install ONLY pytest + build (no maturin, no Rust)
run: python -m pip install --upgrade pip pytest build
- name: Pytest (conformance both-backends + pure package, no _core)
run: python -m pytest tests/test_conformance.py tests/pure tests/edit -v
# Packaged no-native scenario: STAGE only the package files
# (python/ktav/**) plus a staged pyproject.toml whose package-dir
# points INSIDE the stage, build the wheel there — the build can
# never sweep in tests/, spec/ or repo-root files — verify the
# archive contents, then install THAT wheel into a FRESH venv and
# re-run the full five-category corpus through ktav.pure from the
# INSTALLED package (PYTHONPATH empty; ktav._core genuinely absent
# — the pure wheel cannot contain it). Fail-fast order: build →
# verify → install → corpus → semantics.
- name: Stage package files and build the pure wheel from the stage
shell: bash
run: |
set -euo pipefail
# Unique mktemp dir per run — no rm -rf anywhere in staging.
stage="$(mktemp -d)"
mkdir -p "$stage/python/ktav" "$stage/packaging-pure"
cp -R python/ktav/. "$stage/python/ktav/"
cp README.md "$stage/packaging-pure/README.md"
# STAGED pyproject only: rewrite the ../../python mapping to
# point inside the stage; the repo's packaging/pure/pyproject.toml
# is NOT modified.
sed -e 's|../../python/ktav|../python/ktav|' \
-e 's|../../python|../python|' \
packaging/pure/pyproject.toml > "$stage/packaging-pure/pyproject.toml"
python -m build --wheel --outdir dist-pure "$stage/packaging-pure"
- name: Assert wheel filename and version
shell: bash
run: |
set -e
[ -f "dist-pure/ktav-0.8.1-py3-none-any.whl" ] || {
ls dist-pure; exit 1;
}
- name: Verify the pure wheel archive contents before install
shell: bash
run: |
set -euo pipefail
wheel="dist-pure/ktav-0.8.1-py3-none-any.whl"
python scripts/check_pure_wheel_metadata.py "$wheel"
listing="$(python -m zipfile -l "$wheel")"
echo "$listing"
# (a) every pure package member must be present
for member in ktav/__init__.py ktav/pure.py ktav/py.typed ktav/_core.pyi; do
grep -Eq "(^|[[:space:]])${member}([[:space:]]|$)" <<<"$listing" \
|| { echo "missing $member in wheel"; exit 1; }
done
grep -Eq "(^|[[:space:]])ktav/_pure/[^[:space:]]+\.py([[:space:]]|$)" <<<"$listing" \
|| { echo "missing ktav/_pure/*.py in wheel"; exit 1; }
# (b) no compiled or Rust artifacts anywhere
if grep -Eiq "\.(so|pyd|dll|lib)([[:space:]]|$)" <<<"$listing"; then
echo "native/binary artifact inside the pure wheel"; exit 1
fi
# (c) no tests/spec/root-docs paths
if grep -Eq "(^|[[:space:]])(tests|spec|root-docs)/" <<<"$listing"; then
echo "unrelated paths inside the pure wheel"; exit 1
fi
- name: Install the pure wheel into a fresh venv (Windows)
shell: bash
run: |
python -m venv .venv-pure
.venv-pure/Scripts/python -m pip install dist-pure/ktav-0.8.1-py3-none-any.whl
echo "VENV_PY=$(pwd)/.venv-pure/Scripts/python" >> "$GITHUB_ENV"
if: runner.os == 'Windows'
- name: Install the pure wheel into a fresh venv (POSIX)
shell: bash
run: |
python -m venv .venv-pure
.venv-pure/bin/python -m pip install dist-pure/ktav-0.8.1-py3-none-any.whl
echo "VENV_PY=$(pwd)/.venv-pure/bin/python" >> "$GITHUB_ENV"
if: runner.os != 'Windows'
- name: Full corpus through ktav.pure from the installed pure wheel
env:
PYTHONPATH: ""
shell: bash
run: |
"$VENV_PY" scripts/check_installed_pure.py spec/versions/0.8/tests
- name: Installed-wheel semantics (_core absent, native default refuses, version)
env:
PYTHONPATH: ""
shell: bash
run: |
"$VENV_PY" - <<'PY'
import importlib.util
import ktav
assert ktav.__version__ == "0.8.1", ktav.__version__
assert importlib.util.find_spec("ktav._core") is None, (
"pure wheel must not contain ktav._core"
)
try:
ktav.loads("a: 1")
except ImportError as e:
assert "from ktav import pure" in str(e), str(e)
assert "ktav.pure" in str(e), str(e)
else:
raise AssertionError("native default must fail without _core")
from ktav import pure
assert pure.loads("a: 1")["a"] == 1
assert pure.loads(pure.dumps({"port": 8080})) == {"port": 8080}
assert pure.loads_strict(
pure.emit_canonical(pure.loads("b: {a: 1}"))
) == {"b": {"a": 1}}
assert pure.format("a: 1") == "a: 1\n"
print("installed pure wheel OK")
PY
- name: Stage the pure wheel artifact
uses: actions/upload-artifact@v7
with:
name: pure-wheel-${{ matrix.os }}-py${{ matrix.python }}
path: dist-pure
# Guarantees the three OS × Python matrix all pass before merging; useful
# as a required check in branch protection without listing every cell.
test-summary:
name: Test summary
runs-on: ubuntu-latest
needs: [test, pure-mode]
if: always()
steps:
- name: Fail if any matrix job failed
if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
run: exit 1