diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..5e9e40ac --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - labcolors-ephemeral diff --git a/.github/workflows/arb.yml b/.github/workflows/arb.yml new file mode 100644 index 00000000..635ca8b2 --- /dev/null +++ b/.github/workflows/arb.yml @@ -0,0 +1,230 @@ +name: Arb evaluator build and runtime + +on: + workflow_dispatch: + push: + branches: [main] + paths: + - .github/workflows/arb.yml + - crates/labcolors-core/contracts/contextual-region-formula-v1.lcir + - proof/region/v1/** + pull_request: + paths: + - .github/workflows/arb.yml + - crates/labcolors-core/contracts/contextual-region-formula-v1.lcir + - proof/region/v1/** + +permissions: + contents: read + +concurrency: + group: arb-evaluator-build-runtime-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + diagnostic-build-runtime: + name: two offline builds and runtime tests (no artifact) + # Docker is root-equivalent, so this label is provisioned only on a fresh + # one-job VM whose runner group is bound to this exact workflow revision. + runs-on: [self-hosted, Linux, X64, labcolors-ephemeral] + timeout-minutes: 360 + env: + PYTHONDONTWRITEBYTECODE: "1" + PYTHONHASHSEED: "0" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: complete fast Arb contract with exact skip manifest + shell: bash + run: | + set -euo pipefail + python3 proof/region/v1/arb/tests/gate.py + PYTHONOPTIMIZE=2 python3 proof/region/v1/arb/tests/gate.py + + - name: bind run-local native paths after the fast gate + shell: bash + run: | + set -euo pipefail + scope="/sys/fs/cgroup/labcolors-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + binary="$RUNNER_TEMP/arb-native-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + { + echo "LABCOLORS_CGROUP_SCOPE_V1=$scope" + echo "LABCOLORS_EXECUTOR_CGROUP_V1=$scope/proof" + echo "LABCOLORS_ARB_NATIVE_BINARY=$binary" + } >> "$GITHUB_ENV" + + - name: acquire and hash-check exact source archives + shell: bash + run: | + set -euo pipefail + source_dir="$RUNNER_TEMP/arb-source-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + install -d -m 0700 "$source_dir" + echo "LABCOLORS_ARB_SOURCE_DIR=$source_dir" >> "$GITHUB_ENV" + export PYTHONPATH="$GITHUB_WORKSPACE/proof/region/v1" + python3 - <<'PY' > "$source_dir/lock.tsv" + import provenance + + for source in provenance.arb_source_lock_v1().sources: + print( + source.role.name, + source.archive_url, + source.archive_sha256.hex(), + source.archive_length, + sep="\t", + ) + PY + count=0 + while IFS=$'\t' read -r role url digest length; do + archive="$source_dir/${role}.archive" + curl --fail --location --silent --show-error \ + --connect-timeout 30 --max-time 600 --retry 3 --retry-all-errors \ + "$url" --output "$archive" + test "$(stat --format=%s "$archive")" = "$length" + echo "$digest $archive" | sha256sum --check --strict + case "$role" in + GMP) echo "LABCOLORS_GMP_ARCHIVE=$archive" >> "$GITHUB_ENV" ;; + MPFR) echo "LABCOLORS_MPFR_ARCHIVE=$archive" >> "$GITHUB_ENV" ;; + FLINT_ARB) echo "LABCOLORS_FLINT_ARCHIVE=$archive" >> "$GITHUB_ENV" ;; + *) exit 64 ;; + esac + count=$((count + 1)) + done < "$source_dir/lock.tsv" + test "$count" -eq 3 + + - name: acquire the exact pinned OCI manifest + shell: bash + run: | + set -euo pipefail + export PYTHONPATH="$GITHUB_WORKSPACE/proof/region/v1:$GITHUB_WORKSPACE/proof/region/v1/arb" + docker_path="$(realpath "$(command -v docker)")" + test -f "$docker_path" + test ! -L "$docker_path" + image="$(python3 - <<'PY' + import pipeline + print(pipeline.OCI_IMAGE_REFERENCE_V1) + PY + )" + "$docker_path" image inspect "$image" >/dev/null 2>&1 || + /usr/bin/timeout --signal=TERM --kill-after=30s 15m \ + "$docker_path" pull "$image" + echo "LABCOLORS_ARB_PIPELINE_DOCKER=$docker_path" >> "$GITHUB_ENV" + + - name: require the exact diagnostic Docker boundary + shell: bash + run: | + set -euo pipefail + export PYTHONPATH="$GITHUB_WORKSPACE/proof/region/v1:$GITHUB_WORKSPACE/proof/region/v1/arb" + export LABCOLORS_ARB_PIPELINE_DOCKER + python3 - <<'PY' + import os + import sys + from pathlib import Path + + import pipeline + + docker = pipeline.NativeDockerBuildBackendV1( + Path(os.environ["LABCOLORS_ARB_PIPELINE_DOCKER"]) + ).probe() + print(repr(docker)) + if type(docker) is not pipeline.DockerSupportedV1: + sys.exit(78) + PY + + - name: delegate one disposable cgroup subtree + shell: bash + run: | + set -euo pipefail + test -f /proc/sys/kernel/apparmor_restrict_unprivileged_userns + original_userns="$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns)" + case "$original_userns" in + 0|1) ;; + *) exit 78 ;; + esac + echo "LABCOLORS_APPARMOR_USERNS_V1=$original_userns" >> "$GITHUB_ENV" + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + test "$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns)" = 0 + scope="$LABCOLORS_CGROUP_SCOPE_V1" + sudo mkdir "$scope" + sudo chown "$(id -u):$(id -g)" \ + "$scope" \ + "$scope/cgroup.procs" \ + "$scope/cgroup.threads" \ + "$scope/cgroup.subtree_control" + printf '+memory +pids' > "$scope/cgroup.subtree_control" + mkdir "$scope/tasks" "$scope/proof" + printf '+memory +pids' > "$scope/proof/cgroup.subtree_control" + printf '2' > "$scope/proof/pids.max" + mkdir "$scope/proof/observer" + grep --fixed-strings --quiet 'memory' "$scope/proof/cgroup.subtree_control" + grep --fixed-strings --quiet 'pids' "$scope/proof/cgroup.subtree_control" + test "$(cat "$scope/proof/pids.max")" = 2 + + - name: two fresh offline builds and evaluator runtime + shell: bash + run: | + set -euo pipefail + echo "$$" | sudo tee \ + "$LABCOLORS_CGROUP_SCOPE_V1/tasks/cgroup.procs" >/dev/null + python3 proof/region/v1/arb/tests/native_gate.py build + test -f "$LABCOLORS_ARB_NATIVE_BINARY" + test "$(stat --format=%a "$LABCOLORS_ARB_NATIVE_BINARY")" = 400 + + - name: native containment under an atomic two-task subtree + shell: bash + run: | + set -euo pipefail + echo "$$" | sudo tee \ + "$LABCOLORS_EXECUTOR_CGROUP_V1/observer/cgroup.procs" >/dev/null + exec python3 proof/region/v1/arb/tests/native_gate.py executor + + # No upload step: the static binary is an ephemeral observation until a + # linker/member inventory plus notices/source/relink distribution gate exists. + + - name: remove disposable inputs and cgroup + if: always() + shell: bash + run: | + set -uo pipefail + status=0 + record_failure() { + local code="$?" + if (( status == 0 )); then + status="$code" + fi + } + if [[ -n "${LABCOLORS_ARB_SOURCE_DIR:-}" ]]; then + rm -rf -- "$LABCOLORS_ARB_SOURCE_DIR" || record_failure + fi + if [[ -n "${LABCOLORS_ARB_NATIVE_BINARY:-}" ]]; then + rm -f -- "$LABCOLORS_ARB_NATIVE_BINARY" || record_failure + fi + if [[ -n "${LABCOLORS_CGROUP_SCOPE_V1:-}" && \ + -d "$LABCOLORS_CGROUP_SCOPE_V1" ]]; then + if [[ -f "$LABCOLORS_CGROUP_SCOPE_V1/cgroup.kill" ]]; then + echo 1 | sudo tee "$LABCOLORS_CGROUP_SCOPE_V1/cgroup.kill" \ + >/dev/null || record_failure + fi + if [[ -f "$LABCOLORS_CGROUP_SCOPE_V1/cgroup.events" ]]; then + for _ in {1..100}; do + grep --fixed-strings --quiet 'populated 0' \ + "$LABCOLORS_CGROUP_SCOPE_V1/cgroup.events" && break + sleep 0.01 + done + grep --fixed-strings --quiet 'populated 0' \ + "$LABCOLORS_CGROUP_SCOPE_V1/cgroup.events" || record_failure + fi + for child in proof/observer proof tasks; do + if [[ -d "$LABCOLORS_CGROUP_SCOPE_V1/$child" ]]; then + sudo rmdir "$LABCOLORS_CGROUP_SCOPE_V1/$child" || record_failure + fi + done + sudo rmdir "$LABCOLORS_CGROUP_SCOPE_V1" || record_failure + fi + if [[ -n "${LABCOLORS_APPARMOR_USERNS_V1:-}" ]]; then + sudo sysctl -w \ + "kernel.apparmor_restrict_unprivileged_userns=$LABCOLORS_APPARMOR_USERNS_V1" \ + >/dev/null || record_failure + fi + exit "$status" diff --git a/proof/region/v1/PROTOCOL.md b/proof/region/v1/PROTOCOL.md index c180abb7..ee5119a7 100644 --- a/proof/region/v1/PROTOCOL.md +++ b/proof/region/v1/PROTOCOL.md @@ -10,8 +10,10 @@ processes. `region_proof_protocol.py` определяет только structur admission функций сравнения. Текущий `controller.py` безопасно читает и повторно проверяет пять frozen protocol fixtures; он ещё не строит и не запускает evaluator, не разрешает comparator manifest и не создаёт provenance -receipt. Ни один текущий модуль не вычисляет цвет или interval enclosure и не -создаёт semantic proof type. +receipt. Structural protocol и controller не вычисляют formula или interval +enclosure. Диагностический `arb/evaluator` вычисляет Arb-enclosures и выпускает +связанные transcript bytes, но не проверяет их независимым replay и не создаёт +semantic proof type. `V5b2c-0` определяет protocol/admission, но сам не является математическим proof. В c0 нет `DualProofReceiptV1`: structural agreement кодируется @@ -30,11 +32,14 @@ diversity. ## Wire и identity -Все целые беззнаковые и записаны big-endian как `u8`, `u32be` или `u64be`. -`digest` — ровно 32 ненулевых bytes SHA-256. `blob` равен -`u64be(length) || bytes`. Enum занимает один `u8` и принимает только -перечисленные значения. Padding, alignment, reserved fields и trailing bytes -отсутствуют. +Для wire-artifact-ов из `region_proof_protocol.py` все целые беззнаковые и +записаны big-endian как `u8`, `u32be` или `u64be`; `digest` — ровно 32 +ненулевых bytes SHA-256, а `blob` равен `u64be(length) || bytes`. +`SourceReleaseLockV1` и связанные provenance-artifact-ы имеют отдельный codec +в `provenance.py`: его `blob` равен `u32be(length) || bytes`; grammar также +содержит свои `u16` и 20-byte OpenPGP/SHA-1 coordinates. Enum занимает один +`u8` и принимает только перечисленные значения. Padding, alignment, reserved +fields и trailing bytes отсутствуют. До allocation и цикла по records parser проверяет арифметику длины без переполнения, остаток input, точный или минимальный wire-размер всех @@ -53,12 +58,15 @@ artifact, а повторный encode обязан вернуть byte-identica | `ReducedDomainManifestV1` | `LCDOM1\0\0` | `labcolors.proof-region.domain.v1` | | `ProofPolicyV1` | `LCPOL1\0\0` | `labcolors.proof-region.policy.v1` | | `ProofJobV1` | `LCJOB1\0\0` | `labcolors.proof-region.job.v1` | -| `ComparatorManifestV1` | `LCMAN1\0\0` | `labcolors.proof-region.comparator-manifest.v1` | +| `ComparatorManifestV2` | `LCMAN2\0\0` | `labcolors.proof-region.comparator-manifest.v2` | | `DecisionTranscriptV1` | `LCTRN1\0\0` | `labcolors.proof-region.transcript.v1` | | `RunClaimV1` | `LCRUN1\0\0` | `labcolors.proof-region.run-claim.v1` | | `EvaluatorProvenanceClaimV1` | `LCPRV1\0\0` | `labcolors.proof-region.evaluator-provenance-claim.v1` | | `DualComparisonClaimV1` | `LCCMP1\0\0` | `labcolors.proof-region.dual-comparison.v1` | +Версия принадлежит отдельному artifact type. Composite V1 wire связывает +identity независимо версионированного comparator manifest как opaque digest. + ## `ContextualRegionDefinitionV1` Definition не получает protocol magic. Это точный V5b2b canonical preimage: @@ -144,26 +152,78 @@ definition. Job задаёт единственный канонический i вычислителей; только controlled-executor slice сможет доказать отсутствие ambient inputs. Альтернативный JSON/TOML definition запрещён протоколом. -## `ComparatorManifestV1` - -Wire после `LCMAN1\0\0` содержит comparator kind `u8` +## Source lock и integrity observations + +`SourceReleaseLockV1` фиксирует bytes и структурный состав архива. Поле +`.integrity` содержит один `SourceIntegrityPolicyV1`; это требование проверки, +а не заявление о publisher origin. Поле +`legal_files` — только точный project-pinned набор находящихся в архиве legal +files; оно не заявляет полноту legal-набора или compliance распространяемого +бинарника. Несовпадение этого набора имеет отдельную причину +`legal_files_mismatch`. + +Для GMP и MPFR locked detached signature, key packets и исторический +`VALIDSIG` связываются только в +`HistoricalPathRecheckedSignatureDiagnosticV1`. Digest и version запущенного +`gpgv` остаются диагностикой. Запуск принадлежит переданному клиентом +`DiagnosticProcessRunnerV1`: Core ограничивает и парсит возвращённые bytes, но +не выдаёт runner за sandbox, containment или provenance authority. Встроенного +`Popen` fallback нет. Этот тип не устанавливает текущего publisher, +текущий статус или отзыв ключа, происхождение полученных bytes и exact sealed +execution verifier. Такой diagnostic не может заменить будущий source-bound +receipt. + +Для FLINT `GitContentRelationPolicyV1` фиксирует commit, tree, исключённые +paths и отдельные `project_pinned_release_only_files`. `run_git_tree` принимает +такой же client-owned diagnostic runner, после чего Core независимо +пересчитывает commit, commit-to-tree edge, recursive tree и каждый +blob. Поэтому admission создаёт один `RecomputedGitContentRelationV1`: paths +архива должны быть точным дизъюнктным объединением общих Git files и +project-pinned release-only files, а исключённые paths обязаны отсутствовать. +Git executable/version, repository URL и tag являются диагностикой или +координатами поиска и не входят в authority этой relation. Relation доказывает +совпадение content graph, но не publisher или канал получения архива. + +## Diagnostic execution boundary + +`ControlledExecutorV1` — единственный владелец one-shot capability: новый, +неуспешный, перекрывающийся probe или замена backend отзывают ранее выданный +объект до RUN. Capability выпускается контроллером для одного probe-поколения +и одного process id; fork не дублирует право запуска. Backend сообщает только +наблюдённые свойства хоста, получает guard текущего probe и не может продлить +жизнь capability повторно используемым report-объектом. + +Linux backend допускается лишь в отдельном helper process. Helper находится в +прямом дочернем cgroup объявленного parent, а весь parent subtree имеет +`pids.max = 2` и перед probe содержит ровно observer. Эти два task slots имеют +не эвристический смысл: один занимает observer, второй — либо новый thread, +либо единственный controlled child. Kernel pids controller атомарно разрешает +только один из вариантов; поэтому check→fork race не маскируется повторным +опросом `/proc`. Execution child дополнительно получает собственный +`pids.max = 1`, memory limit и `cgroup.kill`; фактические limits читаются назад +до запуска. Отсутствие этой структуры возвращает typed unsupported/setup +outcome. Этот runtime остаётся diagnostic observation и не создаёт receipt. + +## `ComparatorManifestV2` + +Wire после `LCMAN2\0\0` содержит comparator kind `u8` (`1 = Arb`, `2 = MPFI`), затем десять digest coordinates в фиксированном порядке: 1. engine release; 2. upstream source; -3. arithmetic closure; +3. arithmetic input set; 4. wrapper source; 5. evaluator source; 6. build identity, включая compiler, target и exact flags; 7. operation allowlist; -8. test receipt; -9. license closure; +8. test observation; +9. legal file set; 10. exclusions. -Результат wire parse — только raw `ComparatorManifestV1`: его ненулевые +Результат wire parse — только raw `ComparatorManifestV2`: его ненулевые coordinates являются заявленными content addresses, а не доказанным -source binding. `ContentResolvedComparatorManifestV1` создаётся только +source binding. `ContentResolvedComparatorManifestV2` создаётся только после того, как переданный вызывающим `resolve_content_address` для каждой из десяти coordinates вернул exact `bytes` или `Iterable[bytes]`. Сам protocol повторяет SHA-256 по этим bytes/chunks и сравнивает результат с coordinate. Boolean, @@ -331,7 +391,7 @@ raw claim. Он никогда не возвращает admitted candidate. Н refined type. Candidate строится в canonical order Arb → MPFI из двух -`ContentResolvedComparatorManifestV1`, согласованных `RunClaimV1` и +`ContentResolvedComparatorManifestV2`, согласованных `RunClaimV1` и structurally admitted transcripts. Все bindings ведут к одному job, definition, domain и policy; `domain_point_count` равен count связанного manifest. Unresolved counters равны нулю, decision payloads совпадают побайтно, diff --git a/proof/region/v1/arb/build.sh b/proof/region/v1/arb/build.sh new file mode 100755 index 00000000..eea482c1 --- /dev/null +++ b/proof/region/v1/arb/build.sh @@ -0,0 +1,179 @@ +#!/bin/sh +# Build the offline Arb evaluator from already admitted, read-only inputs. +# Acquisition and origin verification intentionally happen before this +# network-free boundary; this recipe never resolves a tool or dependency online. + +set -eu + +if [ "$#" -ne 0 ]; then + printf '%s\n' 'arb build takes no arguments' >&2 + exit 64 +fi + +# Configure and Make observe many ambient variables. Re-exec once from an empty +# environment so a persistent CI host cannot silently change the binary. +if [ "${LC_BUILD_ENV_V1-}" != 1 ]; then + exec /usr/bin/env -i \ + LC_BUILD_ENV_V1=1 \ + PATH=/usr/local/bin:/usr/bin:/bin \ + LC_ALL=C \ + LANG=C \ + TZ=UTC \ + HOME=/nonexistent \ + TMPDIR=/build/tmp \ + SOURCE_DATE_EPOCH=0 \ + ZERO_AR_DATE=1 \ + ARFLAGS=crD \ + /bin/sh "$0" +fi +unset LC_BUILD_ENV_V1 + +umask 022 + +readonly inputs=/inputs +readonly workspace=/workspace +readonly build=/build +readonly output=/out + +require_regular() { + if [ ! -f "$1" ] || [ -L "$1" ]; then + printf 'missing regular build input: %s\n' "$1" >&2 + exit 66 + fi +} + +require_directory() { + if [ ! -d "$1" ] || [ -L "$1" ]; then + printf 'missing normalized source directory: %s\n' "$1" >&2 + exit 66 + fi +} + +require_empty_directory() { + if [ ! -d "$1" ] || [ -L "$1" ]; then + printf 'missing build directory: %s\n' "$1" >&2 + exit 66 + fi + if [ -n "$(find "$1" -mindepth 1 -maxdepth 1 -print -quit)" ]; then + printf 'build directory is not empty: %s\n' "$1" >&2 + exit 65 + fi +} + +require_directory "$inputs/gmp-6.3.0" +require_directory "$inputs/mpfr-4.2.2" +require_directory "$inputs/flint-3.6.0" +require_regular "$inputs/formula.generated.c" +printf '%s %s\n' \ + '9958f20c8ca598625db0593a45f8f8bc79e4b2f22b53263b6c32d78a5e1d2693' \ + "$inputs/formula.generated.c" \ + | /usr/bin/sha256sum --check --strict - +for source in main.c wire.c hash.c interval.c region.c; do + require_regular "$workspace/proof/region/v1/arb/evaluator/$source" +done +require_regular "$workspace/proof/region/v1/arb/evaluator/formula.h" +for header in wire.h hash.h interval.h region.h; do + require_regular "$workspace/proof/region/v1/arb/evaluator/$header" +done +require_empty_directory "$build" +require_empty_directory "$output" + +/usr/bin/mkdir "$build/prefix" "$build/gmp" "$build/mpfr" "$build/flint" "$build/tmp" + +# GCC 15 changed its implicit dialect to GNU C23, where GMP 6.3.0's locked +# no-prototype configure probes have different semantics. GNU C17 is the last +# default those probes targeted; changing it requires a source/toolchain slice +# and a fresh live build, not reliance on a compiler's moving default. +readonly common_cflags='-O2 -g0 -fno-ident -fno-fast-math -ffp-contract=off -fno-lto -std=gnu17 -march=x86-64 -mtune=generic -ffile-prefix-map=/build=. -fdebug-prefix-map=/build=.' +readonly common_ldflags='-Wl,--build-id=none -fno-lto' +readonly prefix="$build/prefix" + +cd "$build/gmp" +ABI=64 CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ + "$inputs/gmp-6.3.0/configure" \ + --build=x86_64-pc-linux-gnu \ + --host=x86_64-pc-linux-gnu \ + --prefix="$prefix" \ + --disable-shared \ + --enable-static \ + --disable-assembly \ + --disable-cxx +/usr/bin/make -j1 +/usr/bin/make check -j1 +/usr/bin/make install + +cd "$build/mpfr" +CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ + "$inputs/mpfr-4.2.2/configure" \ + --build=x86_64-pc-linux-gnu \ + --host=x86_64-pc-linux-gnu \ + --prefix="$prefix" \ + --with-gmp="$prefix" \ + --disable-shared \ + --enable-static \ + --enable-formally-proven-code +/usr/bin/make -j1 +/usr/bin/make check -j1 +/usr/bin/make install + +cd "$build/flint" +CC=/usr/local/bin/gcc CFLAGS="$common_cflags" LDFLAGS="$common_ldflags" \ + "$inputs/flint-3.6.0/configure" \ + --build=x86_64-pc-linux-gnu \ + --host=x86_64-pc-linux-gnu \ + --prefix="$prefix" \ + --with-gmp="$prefix" \ + --with-mpfr="$prefix" \ + --disable-shared \ + --enable-static \ + --disable-assembly \ + --disable-lto \ + --enable-assert +/usr/bin/make -j1 +/usr/bin/make check -j1 +/usr/bin/make install + +cd "$workspace/proof/region/v1/arb/evaluator" +/usr/local/bin/gcc \ + -O2 -g0 -fno-ident -fno-fast-math -ffp-contract=off -fno-lto \ + -march=x86-64 -mtune=generic \ + -ffile-prefix-map=/build=. -fdebug-prefix-map=/build=. \ + -std=c17 -Wall -Wextra -Werror -pedantic \ + -I. -I"$prefix/include" \ + main.c wire.c hash.c interval.c region.c "$inputs/formula.generated.c" \ + -static -Wl,--build-id=none -fno-lto \ + "$prefix/lib/libflint.a" "$prefix/lib/libmpfr.a" "$prefix/lib/libgmp.a" \ + -lm -lpthread \ + -o "$build/arb-evaluator-v1" + +if ! /usr/bin/readelf -l "$build/arb-evaluator-v1" > "$build/program-headers"; then + printf '%s\n' 'cannot inspect evaluator program headers' >&2 + exit 70 +fi +if /usr/bin/grep -q INTERP "$build/program-headers"; then + printf '%s\n' 'evaluator unexpectedly contains PT_INTERP' >&2 + exit 70 +else + grep_status=$? + if [ "$grep_status" -ne 1 ]; then + printf '%s\n' 'cannot search evaluator program headers' >&2 + exit 70 + fi +fi +if ! /usr/bin/readelf -d "$build/arb-evaluator-v1" > "$build/dynamic-section"; then + printf '%s\n' 'cannot inspect evaluator dynamic section' >&2 + exit 70 +fi +if /usr/bin/grep -q NEEDED "$build/dynamic-section"; then + printf '%s\n' 'evaluator unexpectedly contains DT_NEEDED' >&2 + exit 70 +else + grep_status=$? + if [ "$grep_status" -ne 1 ]; then + printf '%s\n' 'cannot search evaluator dynamic section' >&2 + exit 70 + fi +fi + +/usr/bin/install -m 0555 "$build/arb-evaluator-v1" "$output/arb-evaluator-v1" +/usr/bin/sha256sum "$output/arb-evaluator-v1" diff --git a/proof/region/v1/arb/evaluator/formula.h b/proof/region/v1/arb/evaluator/formula.h new file mode 100644 index 00000000..79ccb359 --- /dev/null +++ b/proof/region/v1/arb/evaluator/formula.h @@ -0,0 +1,20 @@ +#ifndef LABCOLOR_ARB_FORMULA_H +#define LABCOLOR_ARB_FORMULA_H + +#include + +#include "interval.h" + +/* Point evaluation owns three output coordinates, so its caller supplies an + array of exactly three initialized Arb elements rather than one arb_t. */ +lc_status lc_formula_point( + arb_ptr output, + const uint8_t rgb[3], + arb_srcptr context, + uint8_t surround, + slong precision +); +lc_status lc_formula_segment(arb_t output, arb_srcptr input, slong precision); +lc_status lc_formula_singleton(arb_t output, arb_srcptr input, slong precision); + +#endif diff --git a/proof/region/v1/arb/evaluator/formula.py b/proof/region/v1/arb/evaluator/formula.py new file mode 100644 index 00000000..90936b7e --- /dev/null +++ b/proof/region/v1/arb/evaluator/formula.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +"""Generate the Arb V1 evaluator from the immutable exact-real SSA.""" + +from __future__ import annotations + +import hashlib +import sys +from dataclasses import dataclass +from pathlib import Path + + +SOURCE_SHA256 = "a6f77ac462f226453b1c27bbd8637b62780b9a640c317a6f50028dacd1de8540" +RELEASE_DOMAIN = b"labcolors.nominal-exact-real-lift.ascii-ssa.v1\0" +RELEASE_SHA256 = "2c626d8ee60eeb62ae4db53660d61bbc25e0efd4e557f0dc1e77565c130b6e52" + +TYPE_DECLARATIONS = ( + "type u8 unsigned_integer_0_255", + "type real mathematical_real", + "type bool exact_boolean", + "type surround_profile closed_enum", +) + +OPERATOR_DECLARATIONS = ( + "operator lookup 2 real table_u8_exact_dyadic_at_ordinal", + "operator eq 2 bool exact_same_type_equality", + "operator select 3 same bool_true_second_else_third", + "operator add 2 real exact_x_plus_y", + "operator sub 2 real exact_x_minus_y", + "operator mul 2 real exact_x_times_y", + "operator div 2 real domain_y_ne_zero_x_div_y_else_domain_unproven", + "operator min 2 real exact_lesser_real", + "operator max 2 real exact_greater_real", + "operator root3 1 real domain_x_ge_zero_unique_y_ge_zero_y_cubed_eq_x_else_domain_unproven", + "operator sqrt 1 real domain_x_ge_zero_unique_y_ge_zero_y_squared_eq_x_else_domain_unproven", + "operator exp 1 real analytic_natural_exponential", + "operator log 1 real domain_x_gt_zero_analytic_natural_logarithm_else_domain_unproven", + "operator sin 1 real analytic_sine_radians", + "operator cos 1 real analytic_cosine_radians", + "operator abs 1 real exact_absolute_value", + "operator sign 1 real negative_minus_one_zero_zero_positive_one", + "operator pow_pos 2 real domain_x_gt_zero_exp_y_mul_log_x_else_domain_unproven", + "operator pow_nn 2 real if_x_eq_zero_and_y_gt_zero_zero_else_pow_pos", + "operator ratio0 2 real if_x_eq_zero_and_y_eq_zero_zero_else_domain_y_gt_zero_x_div_y", +) + +DRIVER_RULES = ( + "rule tone_domain closed_first_last", + "rule out_of_tone_domain outside", + "rule one_knot_tone exact_equality_required", + "rule one_knot_predicate singleton_f_le_zero", + "rule multi_knot_predicate piecewise_linear_segment_f_le_zero", + "rule boundary inclusive", +) + +PROGRAM_INTERFACES = { + "point": ( + (("r8", "u8"), ("g8", "u8"), ("b8", "u8"), + ("adapting_luminance", "real"), ("background_ratio", "real"), + ("surround", "surround_profile")), + 226, + ("jp", "ap", "bp"), + ), + "segment": ( + tuple( + (name, "real") + for name in ( + "segment_t", "segment_a", "segment_b", "segment_t0", + "segment_t1", "segment_c0a", "segment_c0b", "segment_c1a", + "segment_c1b", "segment_rho0", "segment_rho1", "segment_g00", + "segment_g01", "segment_g11", + ) + ), + 27, + ("segment_f",), + ), + "singleton": ( + tuple( + (name, "real") + for name in ( + "singleton_a", "singleton_b", "singleton_ca", "singleton_cb", + "singleton_rho", "singleton_g00", "singleton_g01", "singleton_g11", + ) + ), + 12, + ("singleton_f",), + ), +} + + +class FormulaError(ValueError): + pass + + +@dataclass(frozen=True) +class Node: + name: str + result: str + operator: str + arguments: tuple[str, ...] + + +@dataclass(frozen=True) +class Program: + name: str + inputs: tuple[tuple[str, str], ...] + nodes: tuple[Node, ...] + outputs: tuple[str, ...] + + +@dataclass(frozen=True) +class Formula: + decode: tuple[int, ...] + literals: tuple[tuple[str, int], ...] + enums: tuple[tuple[str, int], ...] + programs: tuple[Program, ...] + + +class Lines: + def __init__(self, values: list[str]): + self.values = values + self.cursor = 0 + + def next(self) -> str: + if self.cursor >= len(self.values): + raise FormulaError(f"unexpected end at line {self.cursor + 1}") + value = self.values[self.cursor] + self.cursor += 1 + return value + + def expect(self, expected: str) -> None: + actual = self.next() + if actual != expected: + raise FormulaError( + f"line {self.cursor}: expected {expected!r}, got {actual!r}" + ) + + +def identifier(value: str) -> bool: + return bool(value) and value[0].islower() and all( + byte.islower() or byte.isdigit() or byte == "_" for byte in value + ) + + +def fields(line: str, count: int) -> tuple[str, ...]: + result = tuple(line.split(" ")) + if len(result) != count: + raise FormulaError(f"record arity {len(result)} != {count}") + return result + + +def finite_bits(token: str) -> int: + if len(token) != 16 or any(byte not in "0123456789abcdef" for byte in token): + raise FormulaError("noncanonical binary64 payload") + bits = int(token, 16) + if bits & 0x7FF0_0000_0000_0000 == 0x7FF0_0000_0000_0000: + raise FormulaError("nonfinite binary64 payload") + if bits == 0x8000_0000_0000_0000: + raise FormulaError("negative zero") + return bits + + +def parse_program(lines: Lines, name: str, globals_: dict[str, str]) -> Program: + expected_inputs, expected_nodes, expected_outputs = PROGRAM_INTERFACES[name] + lines.expect(f"{name}_inputs {len(expected_inputs)}") + symbols = dict(globals_) + inputs: list[tuple[str, str]] = [] + for expected in expected_inputs: + record = fields(lines.next(), 3) + if record != ("input", *expected): + raise FormulaError(f"foreign {name} input") + if record[1] in symbols: + raise FormulaError("shadowed input") + symbols[record[1]] = record[2] + inputs.append((record[1], record[2])) + + lines.expect(f"{name}_nodes {expected_nodes}") + nodes: list[Node] = [] + for _ in range(expected_nodes): + record = tuple(lines.next().split(" ")) + if len(record) < 5 or record[0] != "node" or not identifier(record[1]): + raise FormulaError("invalid node") + node = Node(record[1], record[2], record[3], record[4:]) + validate_node(node, symbols) + if node.name in symbols: + raise FormulaError("shadowed node") + symbols[node.name] = node.result + nodes.append(node) + + if name == "point": + lines.expect("point_checkpoints 39") + checkpoint_names: set[str] = set() + node_names = {node.name for node in nodes} + for _ in range(39): + record = fields(lines.next(), 3) + if ( + record[0] != "checkpoint" + or record[1] in checkpoint_names + or record[2] not in node_names + or symbols[record[2]] != "real" + ): + raise FormulaError("invalid checkpoint") + checkpoint_names.add(record[1]) + + lines.expect(f"{name}_outputs {len(expected_outputs)}") + outputs: list[str] = [] + for expected in expected_outputs: + record = fields(lines.next(), 3) + if record != ("output", expected, "real") or symbols.get(expected) != "real": + raise FormulaError("foreign output") + outputs.append(expected) + return Program(name, tuple(inputs), tuple(nodes), tuple(outputs)) + + +def validate_node(node: Node, symbols: dict[str, str]) -> None: + try: + types = tuple(symbols[value] for value in node.arguments) + except KeyError as error: + raise FormulaError(f"unknown or forward reference {error.args[0]}") from None + unary = {"root3", "sqrt", "exp", "log", "sin", "cos", "abs", "sign"} + binary = {"add", "sub", "mul", "div", "min", "max", "pow_pos", "pow_nn", "ratio0"} + if node.operator == "lookup": + valid = node.result == "real" and types == ("decode_table", "u8") + elif node.operator == "eq": + valid = node.result == "bool" and len(types) == 2 and types[0] == types[1] != "decode_table" + elif node.operator == "select": + valid = len(types) == 3 and types[0] == "bool" and types[1] == types[2] == node.result + elif node.operator in unary: + valid = node.result == "real" and types == ("real",) + elif node.operator in binary: + valid = node.result == "real" and types == ("real", "real") + else: + valid = False + if not valid: + raise FormulaError(f"operator/type mismatch for {node.name}") + + +def parse(source: bytes) -> Formula: + if hashlib.sha256(source).hexdigest() != SOURCE_SHA256: + raise FormulaError("formula source is not the registered V1 content") + release = hashlib.sha256( + RELEASE_DOMAIN + len(source).to_bytes(8, "big") + source + ).hexdigest() + if release != RELEASE_SHA256: + raise FormulaError("formula release mismatch") + if not source.isascii() or not source.endswith(b"\n") or source.endswith(b"\n\n"): + raise FormulaError("formula is not canonical ASCII with one final LF") + text = source.decode("ascii")[:-1] + values = text.split("\n") + for index, line in enumerate(values, 1): + if ( + not line + or line.startswith(" ") + or line.endswith(" ") + or " " in line + or "\t" in line + or "\r" in line + or "#" in line + ): + raise FormulaError(f"line {index} is not canonical") + + lines = Lines(values) + lines.expect("labcolors_exact_real_ssa 1") + lines.expect("arithmetic exact_real_v1") + lines.expect(f"types {len(TYPE_DECLARATIONS)}") + for declaration in TYPE_DECLARATIONS: + lines.expect(declaration) + lines.expect(f"operators {len(OPERATOR_DECLARATIONS)}") + for declaration in OPERATOR_DECLARATIONS: + lines.expect(declaration) + + lines.expect("decode_table decode_srgb8 256") + decode: list[int] = [] + for ordinal in range(256): + record = fields(lines.next(), 3) + if record[:2] != ("decode", f"{ordinal:02x}"): + raise FormulaError("decode order drift") + decode.append(finite_bits(record[2])) + + lines.expect("literals 56") + literals: list[tuple[str, int]] = [] + literal_names: set[str] = set() + literal_values: set[int] = set() + for _ in range(56): + record = fields(lines.next(), 3) + bits = finite_bits(record[2]) + if ( + record[0] != "literal" + or not identifier(record[1]) + or record[1] in literal_names + or bits in literal_values + ): + raise FormulaError("invalid literal") + literal_names.add(record[1]) + literal_values.add(bits) + literals.append((record[1], bits)) + + lines.expect("enum_type surround_profile 3") + enums: list[tuple[str, int]] = [] + for name, tag in (("surround_average", 1), ("surround_dim", 2), ("surround_dark", 3)): + record = fields(lines.next(), 4) + if record != ("enum", "surround_profile", name, f"{tag:02x}"): + raise FormulaError("foreign surround enum") + enums.append((name, tag)) + + globals_: dict[str, str] = {"decode_srgb8": "decode_table"} + globals_.update((name, "real") for name, _ in literals) + globals_.update((name, "surround_profile") for name, _ in enums) + programs = tuple(parse_program(lines, name, globals_) for name in PROGRAM_INTERFACES) + lines.expect(f"driver {len(DRIVER_RULES)}") + for rule in DRIVER_RULES: + lines.expect(rule) + lines.expect("end") + if lines.cursor != len(lines.values): + raise FormulaError("trailing records") + return Formula(tuple(decode), tuple(literals), tuple(enums), programs) + + +def real_expression(name: str, real: dict[str, int]) -> str: + return f"real + {real[name]}" + + +def emit_program(formula: Formula, program: Program) -> list[str]: + real: dict[str, int] = {} + surround: dict[str, str] = {name: str(tag) for name, tag in formula.enums} + boolean: dict[str, str] = {} + lines: list[str] = [] + + for name, _ in formula.literals: + real[name] = len(real) + for name, kind in program.inputs: + if kind == "real": + real[name] = len(real) + elif kind == "surround_profile": + surround[name] = "surround" + + for node in program.nodes: + if node.result == "real": + real[node.name] = len(real) + elif node.result == "bool": + boolean[node.name] = f"condition_{len(boolean)}" + + signature = { + "point": "lc_status lc_formula_point(arb_ptr output, const uint8_t rgb[3], arb_srcptr context, uint8_t surround, slong precision)", + "segment": "lc_status lc_formula_segment(arb_t output, arb_srcptr input, slong precision)", + "singleton": "lc_status lc_formula_singleton(arb_t output, arb_srcptr input, slong precision)", + }[program.name] + lines.extend((signature, "{", " lc_status status = LC_OK;", f" arb_struct real[{len(real)}];")) + for index in range(len(real)): + lines.append(f" arb_init(real + {index});") + for name, bits in formula.literals: + lines.append( + f" status = lc_set_dyadic_bits(real + {real[name]}, UINT64_C(0x{bits:016x}));" + ) + lines.append(" if (status != LC_OK) goto cleanup;") + + real_cursor = 0 + u8_cursor = 0 + u8_values: dict[str, str] = {} + for name, kind in program.inputs: + if kind == "real": + lines.append(f" arb_set(real + {real[name]}, {'context' if program.name == 'point' else 'input'} + {real_cursor});") + real_cursor += 1 + elif kind == "u8": + u8_values[name] = f"rgb[{u8_cursor}]" + u8_cursor += 1 + + adapter = { + "add": "lc_add", "sub": "lc_sub", "mul": "lc_mul", "div": "lc_div", + "min": "lc_min", "max": "lc_max", "root3": "lc_root3", "sqrt": "lc_sqrt", + "exp": "lc_exp", "log": "lc_log", "sin": "lc_sin", "cos": "lc_cos", + "abs": "lc_abs", "sign": "lc_sign", "pow_pos": "lc_pow_pos", + "pow_nn": "lc_pow_nn", "ratio0": "lc_ratio0", + } + for node in program.nodes: + target = real_expression(node.name, real) if node.result == "real" else "" + if node.operator == "lookup": + lines.append( + f" status = lc_set_dyadic_bits({target}, LC_DECODE_BITS[(size_t){u8_values[node.arguments[1]]}]);" + ) + lines.append(" if (status != LC_OK) goto cleanup;") + elif node.operator == "eq": + left = surround[node.arguments[0]] + right = surround[node.arguments[1]] + lines.append(f" int {boolean[node.name]} = ({left} == {right});") + elif node.operator == "select": + condition = boolean[node.arguments[0]] + left = real_expression(node.arguments[1], real) + right = real_expression(node.arguments[2], real) + lines.append(f" arb_set({target}, {condition} ? {left} : {right});") + else: + arguments = ", ".join(real_expression(name, real) for name in node.arguments) + lines.append( + f" status = {adapter[node.operator]}({target}, {arguments}, precision);" + ) + lines.append(" if (status != LC_OK) goto cleanup;") + + for index, name in enumerate(program.outputs): + destination = f"output + {index}" if len(program.outputs) > 1 else "output" + lines.append(f" arb_set({destination}, {real_expression(name, real)});") + lines.append("cleanup:") + for index in range(len(real) - 1, -1, -1): + lines.append(f" arb_clear(real + {index});") + lines.extend((" return status;", "}", "")) + return lines + + +def emit(formula: Formula) -> bytes: + output = [ + "/* Generated from the registered exact-real SSA; do not edit. */", + "#include ", + "#include ", + "#include \"formula.h\"", + "", + "static const uint64_t LC_DECODE_BITS[256] = {", + ] + for index in range(0, 256, 4): + values = ", ".join( + f"UINT64_C(0x{value:016x})" for value in formula.decode[index : index + 4] + ) + output.append(f" {values},") + output.extend(("};", "")) + for program in formula.programs: + output.extend(emit_program(formula, program)) + return ("\n".join(output) + "\n").encode("ascii") + + +def main(argv: list[str]) -> int: + if len(argv) != 2: + print("usage: formula.py FORMULA", file=sys.stderr) + return 2 + try: + source = Path(argv[1]).read_bytes() + generated = emit(parse(source)) + except (OSError, FormulaError) as error: + print(f"formula rejected: {error}", file=sys.stderr) + return 1 + sys.stdout.buffer.write(generated) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/proof/region/v1/arb/evaluator/hash.c b/proof/region/v1/arb/evaluator/hash.c new file mode 100644 index 00000000..20e6543f --- /dev/null +++ b/proof/region/v1/arb/evaluator/hash.c @@ -0,0 +1,173 @@ +#include "hash.h" + +#include + +static const uint32_t round_constants[64] = { + UINT32_C(0x428a2f98), UINT32_C(0x71374491), UINT32_C(0xb5c0fbcf), UINT32_C(0xe9b5dba5), + UINT32_C(0x3956c25b), UINT32_C(0x59f111f1), UINT32_C(0x923f82a4), UINT32_C(0xab1c5ed5), + UINT32_C(0xd807aa98), UINT32_C(0x12835b01), UINT32_C(0x243185be), UINT32_C(0x550c7dc3), + UINT32_C(0x72be5d74), UINT32_C(0x80deb1fe), UINT32_C(0x9bdc06a7), UINT32_C(0xc19bf174), + UINT32_C(0xe49b69c1), UINT32_C(0xefbe4786), UINT32_C(0x0fc19dc6), UINT32_C(0x240ca1cc), + UINT32_C(0x2de92c6f), UINT32_C(0x4a7484aa), UINT32_C(0x5cb0a9dc), UINT32_C(0x76f988da), + UINT32_C(0x983e5152), UINT32_C(0xa831c66d), UINT32_C(0xb00327c8), UINT32_C(0xbf597fc7), + UINT32_C(0xc6e00bf3), UINT32_C(0xd5a79147), UINT32_C(0x06ca6351), UINT32_C(0x14292967), + UINT32_C(0x27b70a85), UINT32_C(0x2e1b2138), UINT32_C(0x4d2c6dfc), UINT32_C(0x53380d13), + UINT32_C(0x650a7354), UINT32_C(0x766a0abb), UINT32_C(0x81c2c92e), UINT32_C(0x92722c85), + UINT32_C(0xa2bfe8a1), UINT32_C(0xa81a664b), UINT32_C(0xc24b8b70), UINT32_C(0xc76c51a3), + UINT32_C(0xd192e819), UINT32_C(0xd6990624), UINT32_C(0xf40e3585), UINT32_C(0x106aa070), + UINT32_C(0x19a4c116), UINT32_C(0x1e376c08), UINT32_C(0x2748774c), UINT32_C(0x34b0bcb5), + UINT32_C(0x391c0cb3), UINT32_C(0x4ed8aa4a), UINT32_C(0x5b9cca4f), UINT32_C(0x682e6ff3), + UINT32_C(0x748f82ee), UINT32_C(0x78a5636f), UINT32_C(0x84c87814), UINT32_C(0x8cc70208), + UINT32_C(0x90befffa), UINT32_C(0xa4506ceb), UINT32_C(0xbef9a3f7), UINT32_C(0xc67178f2), +}; + +static uint32_t +rotate_right(uint32_t value, unsigned distance) +{ + return (value >> distance) | (value << (32U - distance)); +} + +static uint32_t +read_u32_be(const uint8_t *bytes) +{ + return ((uint32_t) bytes[0] << 24) + | ((uint32_t) bytes[1] << 16) + | ((uint32_t) bytes[2] << 8) + | (uint32_t) bytes[3]; +} + +static void +write_u32_be(uint8_t *bytes, uint32_t value) +{ + bytes[0] = (uint8_t) (value >> 24); + bytes[1] = (uint8_t) (value >> 16); + bytes[2] = (uint8_t) (value >> 8); + bytes[3] = (uint8_t) value; +} + +static void +compress(lc_sha256_context *context, const uint8_t block[64]) +{ + uint32_t words[64]; + uint32_t a; + uint32_t b; + uint32_t c; + uint32_t d; + uint32_t e; + uint32_t f; + uint32_t g; + uint32_t h; + + for (size_t index = 0; index < 16; ++index) { + words[index] = read_u32_be(block + index * 4); + } + for (size_t index = 16; index < 64; ++index) { + uint32_t s0 = rotate_right(words[index - 15], 7) + ^ rotate_right(words[index - 15], 18) + ^ (words[index - 15] >> 3); + uint32_t s1 = rotate_right(words[index - 2], 17) + ^ rotate_right(words[index - 2], 19) + ^ (words[index - 2] >> 10); + words[index] = words[index - 16] + s0 + words[index - 7] + s1; + } + + a = context->state[0]; + b = context->state[1]; + c = context->state[2]; + d = context->state[3]; + e = context->state[4]; + f = context->state[5]; + g = context->state[6]; + h = context->state[7]; + for (size_t index = 0; index < 64; ++index) { + uint32_t sum1 = rotate_right(e, 6) ^ rotate_right(e, 11) ^ rotate_right(e, 25); + uint32_t choose = (e & f) ^ ((~e) & g); + uint32_t temporary1 = h + sum1 + choose + round_constants[index] + words[index]; + uint32_t sum0 = rotate_right(a, 2) ^ rotate_right(a, 13) ^ rotate_right(a, 22); + uint32_t majority = (a & b) ^ (a & c) ^ (b & c); + uint32_t temporary2 = sum0 + majority; + + h = g; + g = f; + f = e; + e = d + temporary1; + d = c; + c = b; + b = a; + a = temporary1 + temporary2; + } + context->state[0] += a; + context->state[1] += b; + context->state[2] += c; + context->state[3] += d; + context->state[4] += e; + context->state[5] += f; + context->state[6] += g; + context->state[7] += h; +} + +void +lc_sha256_init(lc_sha256_context *context) +{ + context->state[0] = UINT32_C(0x6a09e667); + context->state[1] = UINT32_C(0xbb67ae85); + context->state[2] = UINT32_C(0x3c6ef372); + context->state[3] = UINT32_C(0xa54ff53a); + context->state[4] = UINT32_C(0x510e527f); + context->state[5] = UINT32_C(0x9b05688c); + context->state[6] = UINT32_C(0x1f83d9ab); + context->state[7] = UINT32_C(0x5be0cd19); + context->bit_length = 0; + context->block_length = 0; +} + +void +lc_sha256_update(lc_sha256_context *context, const uint8_t *bytes, size_t length) +{ + while (length != 0) { + size_t available = sizeof(context->block) - context->block_length; + size_t copied = length < available ? length : available; + + memcpy(context->block + context->block_length, bytes, copied); + context->block_length += copied; + bytes += copied; + length -= copied; + if (context->block_length == sizeof(context->block)) { + compress(context, context->block); + context->bit_length += UINT64_C(512); + context->block_length = 0; + } + } +} + +void +lc_sha256_finish(lc_sha256_context *context, uint8_t digest[32]) +{ + uint64_t total_bits = context->bit_length + (uint64_t) context->block_length * 8U; + + context->block[context->block_length++] = UINT8_C(0x80); + if (context->block_length > 56) { + memset(context->block + context->block_length, 0, 64 - context->block_length); + compress(context, context->block); + context->block_length = 0; + } + memset(context->block + context->block_length, 0, 56 - context->block_length); + for (size_t index = 0; index < 8; ++index) { + context->block[63 - index] = (uint8_t) (total_bits >> (index * 8)); + } + compress(context, context->block); + for (size_t index = 0; index < 8; ++index) { + write_u32_be(digest + index * 4, context->state[index]); + } + memset(context, 0, sizeof(*context)); +} + +void +lc_sha256(const uint8_t *bytes, size_t length, uint8_t digest[32]) +{ + lc_sha256_context context; + + lc_sha256_init(&context); + lc_sha256_update(&context, bytes, length); + lc_sha256_finish(&context, digest); +} diff --git a/proof/region/v1/arb/evaluator/hash.h b/proof/region/v1/arb/evaluator/hash.h new file mode 100644 index 00000000..5fa1ab7e --- /dev/null +++ b/proof/region/v1/arb/evaluator/hash.h @@ -0,0 +1,19 @@ +#ifndef LABCOLOR_ARB_HASH_H +#define LABCOLOR_ARB_HASH_H + +#include +#include + +typedef struct { + uint32_t state[8]; + uint64_t bit_length; + uint8_t block[64]; + size_t block_length; +} lc_sha256_context; + +void lc_sha256_init(lc_sha256_context *context); +void lc_sha256_update(lc_sha256_context *context, const uint8_t *bytes, size_t length); +void lc_sha256_finish(lc_sha256_context *context, uint8_t digest[32]); +void lc_sha256(const uint8_t *bytes, size_t length, uint8_t digest[32]); + +#endif diff --git a/proof/region/v1/arb/evaluator/interval.c b/proof/region/v1/arb/evaluator/interval.c new file mode 100644 index 00000000..a7f9a1e3 --- /dev/null +++ b/proof/region/v1/arb/evaluator/interval.c @@ -0,0 +1,198 @@ +#include "interval.h" + +#include + +lc_status +lc_set_dyadic_bits(arb_t output, uint64_t bits) +{ + uint64_t exponent_bits = (bits >> 52) & UINT64_C(0x7ff); + uint64_t significand = bits & UINT64_C(0x000fffffffffffff); + slong exponent; + fmpz_t integer; + fmpz_t power; + + if (exponent_bits == UINT64_C(0x7ff) || bits == UINT64_C(0x8000000000000000)) { + return LC_INVALID_DYADIC; + } + if (exponent_bits == 0) { + exponent = -1074; + } else { + significand |= UINT64_C(0x0010000000000000); + exponent = (slong) exponent_bits - 1075; + } + + fmpz_init(integer); + fmpz_init(power); + fmpz_set_ui(integer, significand); + if ((bits >> 63) != 0 && significand != 0) { + fmpz_neg(integer, integer); + } + fmpz_set_si(power, exponent); + arb_set_fmpz_2exp(output, integer, power); + fmpz_clear(power); + fmpz_clear(integer); + return LC_OK; +} + +void +lc_interval_get_dyadic_bounds( + fmpz_t lower, + fmpz_t upper, + fmpz_t exponent, + arb_srcptr value +) +{ + arb_get_interval_fmpz_2exp(lower, upper, exponent, value); +} + +lc_status +lc_add(arb_t output, arb_srcptr left, arb_srcptr right, slong precision) +{ + arb_add(output, left, right, precision); + return LC_OK; +} + +lc_status +lc_sub(arb_t output, arb_srcptr left, arb_srcptr right, slong precision) +{ + arb_sub(output, left, right, precision); + return LC_OK; +} + +lc_status +lc_mul(arb_t output, arb_srcptr left, arb_srcptr right, slong precision) +{ + arb_mul(output, left, right, precision); + return LC_OK; +} + +lc_status +lc_div(arb_t output, arb_srcptr left, arb_srcptr right, slong precision) +{ + if (arb_contains_zero(right)) { + return LC_DOMAIN_UNPROVEN; + } + arb_div(output, left, right, precision); + return LC_OK; +} + +lc_status +lc_min(arb_t output, arb_srcptr left, arb_srcptr right, slong precision) +{ + arb_min(output, left, right, precision); + return LC_OK; +} + +lc_status +lc_max(arb_t output, arb_srcptr left, arb_srcptr right, slong precision) +{ + arb_max(output, left, right, precision); + return LC_OK; +} + +lc_status +lc_root3(arb_t output, arb_srcptr input, slong precision) +{ + if (!arb_is_nonnegative(input)) { + return LC_DOMAIN_UNPROVEN; + } + arb_root_ui(output, input, 3, precision); + return LC_OK; +} + +lc_status +lc_sqrt(arb_t output, arb_srcptr input, slong precision) +{ + if (!arb_is_nonnegative(input)) { + return LC_DOMAIN_UNPROVEN; + } + arb_sqrt(output, input, precision); + return LC_OK; +} + +lc_status +lc_exp(arb_t output, arb_srcptr input, slong precision) +{ + arb_exp(output, input, precision); + return LC_OK; +} + +lc_status +lc_log(arb_t output, arb_srcptr input, slong precision) +{ + if (!arb_is_positive(input)) { + return LC_DOMAIN_UNPROVEN; + } + arb_log(output, input, precision); + return LC_OK; +} + +lc_status +lc_sin(arb_t output, arb_srcptr input, slong precision) +{ + arb_sin(output, input, precision); + return LC_OK; +} + +lc_status +lc_cos(arb_t output, arb_srcptr input, slong precision) +{ + arb_cos(output, input, precision); + return LC_OK; +} + +lc_status +lc_abs(arb_t output, arb_srcptr input, slong precision) +{ + (void) precision; + arb_abs(output, input); + return LC_OK; +} + +lc_status +lc_sign(arb_t output, arb_srcptr input, slong precision) +{ + (void) precision; + arb_sgn(output, input); + return LC_OK; +} + +lc_status +lc_pow_pos(arb_t output, arb_srcptr base, arb_srcptr exponent, slong precision) +{ + arb_t logarithm; + + if (!arb_is_positive(base)) { + return LC_DOMAIN_UNPROVEN; + } + arb_init(logarithm); + arb_log(logarithm, base, precision); + arb_mul(logarithm, logarithm, exponent, precision); + arb_exp(output, logarithm, precision); + arb_clear(logarithm); + return LC_OK; +} + +lc_status +lc_pow_nn(arb_t output, arb_srcptr base, arb_srcptr exponent, slong precision) +{ + if (arb_is_zero(base) && arb_is_positive(exponent)) { + arb_zero(output); + return LC_OK; + } + return lc_pow_pos(output, base, exponent, precision); +} + +lc_status +lc_ratio0(arb_t output, arb_srcptr numerator, arb_srcptr denominator, slong precision) +{ + if (arb_is_zero(numerator) && arb_is_zero(denominator)) { + arb_zero(output); + return LC_OK; + } + if (!arb_is_positive(denominator)) { + return LC_DOMAIN_UNPROVEN; + } + arb_div(output, numerator, denominator, precision); + return LC_OK; +} diff --git a/proof/region/v1/arb/evaluator/interval.h b/proof/region/v1/arb/evaluator/interval.h new file mode 100644 index 00000000..88c936e1 --- /dev/null +++ b/proof/region/v1/arb/evaluator/interval.h @@ -0,0 +1,41 @@ +#ifndef LABCOLOR_ARB_INTERVAL_H +#define LABCOLOR_ARB_INTERVAL_H + +#include + +#include +#include + +typedef enum { + LC_OK = 0, + LC_DOMAIN_UNPROVEN = 1, + LC_INVALID_DYADIC = 2 +} lc_status; + +lc_status lc_set_dyadic_bits(arb_t output, uint64_t bits); +void lc_interval_get_dyadic_bounds( + fmpz_t lower, + fmpz_t upper, + fmpz_t exponent, + arb_srcptr value +); + +lc_status lc_add(arb_t output, arb_srcptr left, arb_srcptr right, slong precision); +lc_status lc_sub(arb_t output, arb_srcptr left, arb_srcptr right, slong precision); +lc_status lc_mul(arb_t output, arb_srcptr left, arb_srcptr right, slong precision); +lc_status lc_div(arb_t output, arb_srcptr left, arb_srcptr right, slong precision); +lc_status lc_min(arb_t output, arb_srcptr left, arb_srcptr right, slong precision); +lc_status lc_max(arb_t output, arb_srcptr left, arb_srcptr right, slong precision); +lc_status lc_root3(arb_t output, arb_srcptr input, slong precision); +lc_status lc_sqrt(arb_t output, arb_srcptr input, slong precision); +lc_status lc_exp(arb_t output, arb_srcptr input, slong precision); +lc_status lc_log(arb_t output, arb_srcptr input, slong precision); +lc_status lc_sin(arb_t output, arb_srcptr input, slong precision); +lc_status lc_cos(arb_t output, arb_srcptr input, slong precision); +lc_status lc_abs(arb_t output, arb_srcptr input, slong precision); +lc_status lc_sign(arb_t output, arb_srcptr input, slong precision); +lc_status lc_pow_pos(arb_t output, arb_srcptr base, arb_srcptr exponent, slong precision); +lc_status lc_pow_nn(arb_t output, arb_srcptr base, arb_srcptr exponent, slong precision); +lc_status lc_ratio0(arb_t output, arb_srcptr numerator, arb_srcptr denominator, slong precision); + +#endif diff --git a/proof/region/v1/arb/evaluator/main.c b/proof/region/v1/arb/evaluator/main.c new file mode 100644 index 00000000..262f612c --- /dev/null +++ b/proof/region/v1/arb/evaluator/main.c @@ -0,0 +1,516 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "hash.h" +#include "wire.h" + +typedef struct { + uint8_t *bytes; + size_t length; + size_t capacity; +} byte_buffer; + +static const uint8_t transcript_magic[8] = "LCTRN1\0"; +static const uint8_t accounting_domain[] = "labcolors.arb-evaluation-accounting.v1\0"; +static const uint8_t exact_trace_domain[] = + "labcolors.proof-region.exact-zero-signal-trace.v1\0"; +static const uint8_t boundary_enclosure_domain[] = + "labcolors.arb-boundary-enclosure.v1\0"; + +static void +buffer_clear(byte_buffer *buffer) +{ + free(buffer->bytes); + memset(buffer, 0, sizeof(*buffer)); +} + +static bool +buffer_reserve(byte_buffer *buffer, size_t additional) +{ + size_t required; + size_t capacity; + uint8_t *replacement; + + if (additional > SIZE_MAX - buffer->length) { + return false; + } + required = buffer->length + additional; + if (required <= buffer->capacity) { + return required == 0 || buffer->bytes != NULL; + } + capacity = buffer->capacity == 0 ? 4096 : buffer->capacity; + while (capacity < required) { + if (capacity > SIZE_MAX / 2) { + capacity = required; + break; + } + capacity *= 2; + } + replacement = realloc(buffer->bytes, capacity); + if (replacement == NULL) { + return false; + } + buffer->bytes = replacement; + buffer->capacity = capacity; + return true; +} + +static bool +buffer_append(byte_buffer *buffer, const uint8_t *bytes, size_t length) +{ + if (length == 0) { + return true; + } + if (!buffer_reserve(buffer, length)) { + return false; + } + memcpy(buffer->bytes + buffer->length, bytes, length); + buffer->length += length; + return true; +} + +static bool +buffer_u8(byte_buffer *buffer, uint8_t value) +{ + return buffer_append(buffer, &value, 1); +} + +static bool +buffer_u32(byte_buffer *buffer, uint32_t value) +{ + uint8_t bytes[4]; + + lc_write_u32_be(bytes, value); + return buffer_append(buffer, bytes, sizeof(bytes)); +} + +static bool +buffer_u64(byte_buffer *buffer, uint64_t value) +{ + uint8_t bytes[8]; + + lc_write_u64_be(bytes, value); + return buffer_append(buffer, bytes, sizeof(bytes)); +} + +static bool +read_stdin(byte_buffer *input) +{ + uint8_t chunk[16384]; + + for (;;) { + ssize_t count = read(STDIN_FILENO, chunk, sizeof(chunk)); + + if (count < 0) { + if (errno == EINTR) { + continue; + } + return false; + } + if (count == 0) { + return input->length != 0; + } + if (!buffer_append(input, chunk, (size_t) count)) { + return false; + } + } +} + +static bool +digest_is_nonzero(const uint8_t digest[32]) +{ + uint8_t aggregate = 0; + + for (size_t index = 0; index < 32; ++index) { + aggregate |= digest[index]; + } + return aggregate != 0; +} + +static bool +parse_manifest_identity(const char *text, uint8_t identity[32]) +{ + uint8_t aggregate = 0; + + if (strlen(text) != 64) { + return false; + } + for (size_t index = 0; index < 32; ++index) { + uint8_t value = 0; + + for (size_t nibble = 0; nibble < 2; ++nibble) { + unsigned char character = (unsigned char) text[index * 2 + nibble]; + + value <<= 4; + if (character >= '0' && character <= '9') { + value |= (uint8_t) (character - '0'); + } else if (character >= 'a' && character <= 'f') { + value |= (uint8_t) (character - 'a' + 10); + } else { + return false; + } + } + identity[index] = value; + aggregate |= value; + } + return aggregate != 0; +} + +static bool +exact_trace_digest( + const lc_job *job, + uint32_t ordinal, + const lc_region_result *result, + uint8_t digest[32] +) +{ + lc_sha256_context context; + uint8_t encoded_ordinal[4]; + uint8_t encoded_branch[8]; + + /* + * Precision and enclosure belong to an engine run, not to the exact + * signal. Job, ordinal and the first exact branch select one replayable + * mathematical trace identically for Arb and an independent comparator. + */ + lc_write_u32_be(encoded_ordinal, ordinal); + lc_write_u64_be(encoded_branch, result->exact_branch); + lc_sha256_init(&context); + lc_sha256_update(&context, exact_trace_domain, sizeof(exact_trace_domain) - 1); + lc_sha256_update(&context, job->job_identity, 32); + lc_sha256_update(&context, encoded_ordinal, sizeof(encoded_ordinal)); + lc_sha256_update(&context, encoded_branch, sizeof(encoded_branch)); + lc_sha256_finish(&context, digest); + return digest_is_nonzero(digest); +} + +static bool +boundary_enclosure_digest( + const lc_job *job, + uint32_t ordinal, + uint32_t precision, + const lc_region_result *result, + uint8_t digest[32] +) +{ + lc_sha256_context context; + uint8_t encoded[9]; + fmpz_t lower; + fmpz_t upper; + fmpz_t exponent; + char *lower_text = NULL; + char *upper_text = NULL; + char *exponent_text = NULL; + bool success = false; + + lc_write_u32_be(encoded, ordinal); + lc_write_u32_be(encoded + 4, precision); + encoded[8] = (uint8_t) result->formula_status; + lc_sha256_init(&context); + lc_sha256_update( + &context, + boundary_enclosure_domain, + sizeof(boundary_enclosure_domain) - 1 + ); + lc_sha256_update(&context, job->job_identity, 32); + lc_sha256_update(&context, encoded, sizeof(encoded)); + encoded[0] = result->has_enclosure ? 1 : 0; + lc_sha256_update(&context, encoded, 1); + if (result->has_enclosure) { + uint8_t length[8]; + + fmpz_init(lower); + fmpz_init(upper); + fmpz_init(exponent); + lc_interval_get_dyadic_bounds(lower, upper, exponent, &result->enclosure); + lower_text = fmpz_get_str(NULL, 16, lower); + upper_text = fmpz_get_str(NULL, 16, upper); + exponent_text = fmpz_get_str(NULL, 16, exponent); + if (lower_text == NULL || upper_text == NULL || exponent_text == NULL) { + goto cleanup; + } + const char *values[3] = {lower_text, upper_text, exponent_text}; + for (size_t index = 0; index < 3; ++index) { + size_t text_length = strlen(values[index]); + + lc_write_u64_be(length, (uint64_t) text_length); + lc_sha256_update(&context, length, sizeof(length)); + lc_sha256_update(&context, (const uint8_t *) values[index], text_length); + } + } + lc_sha256_finish(&context, digest); + success = digest_is_nonzero(digest); + +cleanup: + if (result->has_enclosure) { + flint_free(exponent_text); + flint_free(upper_text); + flint_free(lower_text); + fmpz_clear(exponent); + fmpz_clear(upper); + fmpz_clear(lower); + } + return success; +} + +static void +account_point( + lc_sha256_context *accounting, + uint32_t ordinal, + uint32_t precision, + uint64_t consumed, + lc_region_outcome outcome +) +{ + uint8_t record[17]; + + lc_write_u32_be(record, ordinal); + lc_write_u32_be(record + 4, precision); + lc_write_u64_be(record + 8, consumed); + record[16] = (uint8_t) outcome; + lc_sha256_update(accounting, record, sizeof(record)); +} + +static bool +append_digest_witness( + byte_buffer *witnesses, + uint8_t kind, + uint32_t ordinal, + const uint8_t digest[32] +) +{ + return buffer_u8(witnesses, kind) + && buffer_u32(witnesses, ordinal) + && buffer_append(witnesses, digest, 32); +} + +static bool +append_resource_witness( + byte_buffer *witnesses, + uint32_t ordinal, + uint8_t scope, + uint64_t grant +) +{ + return buffer_u8(witnesses, 3) + && buffer_u32(witnesses, ordinal) + && buffer_u8(witnesses, scope) + && buffer_u64(witnesses, grant) + && buffer_u64(witnesses, grant); +} + +static uint64_t +lesser_u64(uint64_t left, uint64_t right) +{ + return left < right ? left : right; +} + +static bool +evaluate( + const lc_job *job, + const uint8_t comparator_identity[32], + byte_buffer *output +) +{ + byte_buffer decisions = {0}; + byte_buffer witnesses = {0}; + lc_domain_iterator iterator; + lc_region_result result; + lc_sha256_context accounting; + uint64_t counters[4] = {0, 0, 0, 0}; + uint64_t equality_count = 0; + uint64_t witness_count = 0; + uint64_t global_remaining = job->policy.global_pregrant; + uint8_t accounting_digest[32]; + size_t decision_length; + bool success = false; + + if (job->domain.point_count == 0 + || job->policy.precision_count == 0 + || job->domain.point_count > SIZE_MAX - 3) { + return false; + } + decision_length = ((size_t) job->domain.point_count + 3) / 4; + if (decision_length == 0 + || !buffer_reserve(&decisions, decision_length) + || decisions.bytes == NULL) { + return false; + } + memset(decisions.bytes, 0, decision_length); + decisions.length = decision_length; + lc_sha256_init(&accounting); + lc_sha256_update(&accounting, accounting_domain, sizeof(accounting_domain) - 1); + lc_sha256_update(&accounting, job->job_identity, 32); + lc_sha256_update(&accounting, job->domain.identity, 32); + lc_sha256_update(&accounting, job->policy.identity, 32); + lc_sha256_update(&accounting, comparator_identity, 32); + lc_region_result_init(&result); + lc_domain_iterator_init(&iterator, &job->domain); + for (uint64_t point_index = 0; point_index < job->domain.point_count; ++point_index) { + uint64_t point_grant = lesser_u64( + job->policy.per_point_work, + global_remaining + ); + uint64_t point_remaining = point_grant; + uint64_t point_consumed = 0; + uint8_t resource_scope = job->policy.per_point_work <= global_remaining + ? 1 + : 2; + uint32_t ordinal; + uint32_t final_precision = job->policy.precision_ladder[0]; + uint8_t rgb[3]; + + /* A point owns its ordinal-prefix pregrant even when it uses none. */ + global_remaining -= point_grant; + if (!lc_domain_iterator_next(&iterator, &ordinal)) { + goto cleanup; + } + lc_ordinal_to_rgb(ordinal, rgb); + for (size_t rung = 0; rung < job->policy.precision_count; ++rung) { + uint64_t grant = point_remaining; + + final_precision = job->policy.precision_ladder[rung]; + lc_region_evaluate_rgb( + &result, + rgb, + job->context, + job->surround, + &job->region, + (slong) final_precision, + grant + ); + if (result.consumed_branches > grant + || result.consumed_branches > point_remaining) { + goto cleanup; + } + point_remaining -= result.consumed_branches; + point_consumed += result.consumed_branches; + if (result.outcome != LC_REGION_BOUNDARY_UNPROVEN) { + break; + } + } + if ((unsigned) result.outcome > LC_REGION_RESOURCE_LIMIT_REACHED) { + goto cleanup; + } + decisions.bytes[point_index / 4] |= (uint8_t) result.outcome + << (6U - 2U * (unsigned) (point_index % 4)); + ++counters[result.outcome]; + account_point(&accounting, ordinal, final_precision, point_consumed, result.outcome); + if (result.outcome == LC_REGION_INSIDE && result.exact_boundary) { + uint8_t digest[32]; + + if (!exact_trace_digest(job, ordinal, &result, digest) + || !append_digest_witness(&witnesses, 1, ordinal, digest)) { + goto cleanup; + } + ++equality_count; + ++witness_count; + } else if (result.outcome == LC_REGION_BOUNDARY_UNPROVEN) { + uint8_t digest[32]; + + if (!boundary_enclosure_digest( + job, + ordinal, + final_precision, + &result, + digest + ) + || !append_digest_witness(&witnesses, 2, ordinal, digest)) { + goto cleanup; + } + ++witness_count; + } else if (result.outcome == LC_REGION_RESOURCE_LIMIT_REACHED) { + if (point_consumed != point_grant + || !append_resource_witness( + &witnesses, + ordinal, + resource_scope, + point_grant + )) { + goto cleanup; + } + ++witness_count; + } + } + lc_sha256_finish(&accounting, accounting_digest); + if (!digest_is_nonzero(accounting_digest) + || !buffer_append(output, transcript_magic, sizeof(transcript_magic)) + || !buffer_append(output, job->job_identity, 32) + || !buffer_append(output, job->domain.identity, 32) + || !buffer_append(output, comparator_identity, 32) + || !buffer_u64(output, job->domain.point_count) + || !buffer_u64(output, decisions.length) + || !buffer_append(output, decisions.bytes, decisions.length)) { + goto cleanup; + } + for (size_t index = 0; index < 4; ++index) { + if (!buffer_u64(output, counters[index])) { + goto cleanup; + } + } + if (!buffer_u64(output, equality_count) + || !buffer_append(output, accounting_digest, 32) + || !buffer_u64(output, witness_count) + || !buffer_append(output, witnesses.bytes, witnesses.length)) { + goto cleanup; + } + success = true; + +cleanup: + lc_region_result_clear(&result); + buffer_clear(&witnesses); + buffer_clear(&decisions); + return success; +} + +int +main(int argc, char **argv) +{ + byte_buffer input = {0}; + byte_buffer output = {0}; + lc_job job; + lc_wire_error error; + uint8_t comparator_identity[32]; + int status = 1; + + if (argc != 5 + || strcmp(argv[1], "--manifest-identity") != 0 + || !parse_manifest_identity(argv[2], comparator_identity) + || strcmp(argv[3], "--job") != 0 + || strcmp(argv[4], "/dev/stdin") != 0) { + fputs( + "usage: arb-evaluator --manifest-identity HEX64 --job /dev/stdin\n", + stderr + ); + return 64; + } + if (!read_stdin(&input)) { + fputs("job read failed\n", stderr); + goto cleanup_input; + } + if (!lc_parse_job(&job, input.bytes, input.length, &error)) { + fprintf(stderr, "job rejected: %s\n", lc_wire_error_name(error)); + goto cleanup_input; + } + if (!evaluate(&job, comparator_identity, &output)) { + fputs("evaluation failed\n", stderr); + goto cleanup_job; + } + if (!lc_write_all(STDOUT_FILENO, output.bytes, output.length)) { + fputs("result write failed\n", stderr); + goto cleanup_job; + } + status = 0; + +cleanup_job: + buffer_clear(&output); + lc_job_clear(&job); +cleanup_input: + buffer_clear(&input); + return status; +} diff --git a/proof/region/v1/arb/evaluator/region.c b/proof/region/v1/arb/evaluator/region.c new file mode 100644 index 00000000..ada1334c --- /dev/null +++ b/proof/region/v1/arb/evaluator/region.c @@ -0,0 +1,287 @@ +#include "region.h" + +#include + +#include "formula.h" + +static void +reset_result(lc_region_result *result) +{ + result->outcome = LC_REGION_BOUNDARY_UNPROVEN; + result->formula_status = LC_OK; + result->exact_boundary = false; + result->has_enclosure = false; + result->exact_branch = 0; + result->consumed_branches = 0; + arb_zero(&result->enclosure); +} + +static void +record_enclosure(lc_region_result *result, arb_srcptr value, slong precision) +{ + if (result->has_enclosure) { + arb_union(&result->enclosure, &result->enclosure, value, precision); + } else { + arb_set(&result->enclosure, value); + result->has_enclosure = true; + } +} + +bool +lc_region_init(lc_region *region, size_t knot_count) +{ + region->knots = NULL; + region->knot_count = 0; + arb_init(®ion->metric_aa); + arb_init(®ion->metric_ab); + arb_init(®ion->metric_bb); + if (knot_count == 0 || knot_count > SIZE_MAX / sizeof(*region->knots)) { + lc_region_clear(region); + return false; + } + region->knots = calloc(knot_count, sizeof(*region->knots)); + if (region->knots == NULL) { + lc_region_clear(region); + return false; + } + region->knot_count = knot_count; + for (size_t index = 0; index < knot_count; ++index) { + arb_init(®ion->knots[index].tone); + arb_init(®ion->knots[index].center_a); + arb_init(®ion->knots[index].center_b); + arb_init(®ion->knots[index].radius_squared); + } + return true; +} + +void +lc_region_clear(lc_region *region) +{ + if (region->knots != NULL) { + for (size_t index = 0; index < region->knot_count; ++index) { + arb_clear(®ion->knots[index].radius_squared); + arb_clear(®ion->knots[index].center_b); + arb_clear(®ion->knots[index].center_a); + arb_clear(®ion->knots[index].tone); + } + free(region->knots); + } + arb_clear(®ion->metric_bb); + arb_clear(®ion->metric_ab); + arb_clear(®ion->metric_aa); + region->knots = NULL; + region->knot_count = 0; +} + +void +lc_region_result_init(lc_region_result *result) +{ + arb_init(&result->enclosure); + reset_result(result); +} + +void +lc_region_result_clear(lc_region_result *result) +{ + arb_clear(&result->enclosure); +} + +static void +evaluate_singleton( + lc_region_result *result, + arb_srcptr point, + const lc_region *region, + slong precision, + uint64_t branch_grant +) +{ + arb_struct input[8]; + arb_t predicate; + + if (!arb_equal(point, ®ion->knots[0].tone)) { + result->outcome = arb_overlaps(point, ®ion->knots[0].tone) + ? LC_REGION_BOUNDARY_UNPROVEN + : LC_REGION_OUTSIDE; + return; + } + if (branch_grant == 0) { + result->outcome = LC_REGION_RESOURCE_LIMIT_REACHED; + return; + } + for (size_t index = 0; index < 8; ++index) { + arb_init(input + index); + } + arb_set(input + 0, point + 1); + arb_set(input + 1, point + 2); + arb_set(input + 2, ®ion->knots[0].center_a); + arb_set(input + 3, ®ion->knots[0].center_b); + arb_set(input + 4, ®ion->knots[0].radius_squared); + arb_set(input + 5, ®ion->metric_aa); + arb_set(input + 6, ®ion->metric_ab); + arb_set(input + 7, ®ion->metric_bb); + arb_init(predicate); + result->formula_status = lc_formula_singleton(predicate, input, precision); + result->consumed_branches = 1; + if (result->formula_status == LC_OK) { + record_enclosure(result, predicate, precision); + if (arb_is_nonpositive(predicate)) { + result->outcome = LC_REGION_INSIDE; + result->exact_boundary = arb_is_zero(predicate); + result->exact_branch = 0; + } else if (arb_is_positive(predicate)) { + result->outcome = LC_REGION_OUTSIDE; + } + } + arb_clear(predicate); + for (size_t index = 8; index-- != 0;) { + arb_clear(input + index); + } +} + +void +lc_region_decide( + lc_region_result *result, + arb_srcptr point, + const lc_region *region, + slong precision, + uint64_t branch_grant +) +{ + bool any_segment = false; + bool all_inside = true; + bool all_outside = true; + bool exact_zero = false; + bool outside_possible; + uint64_t exact_branch = 0; + arb_t segment_domain; + arb_t intersection; + + reset_result(result); + /* FLINT's two-bit minimum applies to the public decision entry point too; + otherwise a singleton can bypass the policy before any segment exists. */ + if (precision < 2) { + result->formula_status = LC_DOMAIN_UNPROVEN; + return; + } + if (region->knot_count == 1) { + evaluate_singleton(result, point, region, precision, branch_grant); + return; + } + if (region->knot_count < 2) { + result->formula_status = LC_DOMAIN_UNPROVEN; + return; + } + if (arb_lt(point, ®ion->knots[0].tone) + || arb_gt(point, ®ion->knots[region->knot_count - 1].tone)) { + result->outcome = LC_REGION_OUTSIDE; + return; + } + outside_possible = !arb_ge(point, ®ion->knots[0].tone) + || !arb_le(point, ®ion->knots[region->knot_count - 1].tone); + arb_init(segment_domain); + arb_init(intersection); + for (size_t index = 0; index + 1 < region->knot_count; ++index) { + const lc_region_knot *left = region->knots + index; + const lc_region_knot *right = region->knots + index + 1; + arb_struct input[14]; + arb_t predicate; + + arb_union(segment_domain, &left->tone, &right->tone, precision); + if (!arb_intersection(intersection, point, segment_domain, precision)) { + continue; + } + any_segment = true; + if (result->consumed_branches == branch_grant) { + result->outcome = LC_REGION_RESOURCE_LIMIT_REACHED; + goto cleanup; + } + for (size_t input_index = 0; input_index < 14; ++input_index) { + arb_init(input + input_index); + } + arb_set(input + 0, intersection); + arb_set(input + 1, point + 1); + arb_set(input + 2, point + 2); + arb_set(input + 3, &left->tone); + arb_set(input + 4, &right->tone); + arb_set(input + 5, &left->center_a); + arb_set(input + 6, &left->center_b); + arb_set(input + 7, &right->center_a); + arb_set(input + 8, &right->center_b); + arb_set(input + 9, &left->radius_squared); + arb_set(input + 10, &right->radius_squared); + arb_set(input + 11, ®ion->metric_aa); + arb_set(input + 12, ®ion->metric_ab); + arb_set(input + 13, ®ion->metric_bb); + arb_init(predicate); + result->formula_status = lc_formula_segment(predicate, input, precision); + ++result->consumed_branches; + if (result->formula_status == LC_OK) { + bool inside = arb_is_nonpositive(predicate); + bool outside = arb_is_positive(predicate); + bool branch_exact = arb_is_zero(predicate); + + record_enclosure(result, predicate, precision); + all_inside = all_inside && inside; + all_outside = all_outside && outside; + /* Strict segment order makes the first exact branch canonical. */ + if (branch_exact && !exact_zero) { + exact_branch = (uint64_t) index; + } + exact_zero = exact_zero || branch_exact; + } else { + all_inside = false; + all_outside = false; + } + arb_clear(predicate); + for (size_t input_index = 14; input_index-- != 0;) { + arb_clear(input + input_index); + } + } + if (!any_segment) { + result->outcome = LC_REGION_BOUNDARY_UNPROVEN; + } else if (all_outside) { + result->outcome = LC_REGION_OUTSIDE; + } else if (all_inside && !outside_possible) { + result->outcome = LC_REGION_INSIDE; + result->exact_boundary = exact_zero; + result->exact_branch = exact_branch; + } else { + result->outcome = LC_REGION_BOUNDARY_UNPROVEN; + } + +cleanup: + arb_clear(intersection); + arb_clear(segment_domain); +} + +void +lc_region_evaluate_rgb( + lc_region_result *result, + const uint8_t rgb[3], + arb_srcptr context, + uint8_t surround, + const lc_region *region, + slong precision, + uint64_t branch_grant +) +{ + arb_struct point[3]; + + reset_result(result); + /* FLINT defines two bits as its minimum working precision. Lower policy + rungs remain unresolved and must never enter Arb arithmetic. */ + if (precision < 2) { + result->formula_status = LC_DOMAIN_UNPROVEN; + return; + } + for (size_t index = 0; index < 3; ++index) { + arb_init(point + index); + } + result->formula_status = lc_formula_point(point, rgb, context, surround, precision); + if (result->formula_status == LC_OK) { + lc_region_decide(result, point, region, precision, branch_grant); + } + for (size_t index = 3; index-- != 0;) { + arb_clear(point + index); + } +} diff --git a/proof/region/v1/arb/evaluator/region.h b/proof/region/v1/arb/evaluator/region.h new file mode 100644 index 00000000..7a2252ed --- /dev/null +++ b/proof/region/v1/arb/evaluator/region.h @@ -0,0 +1,63 @@ +#ifndef LABCOLOR_ARB_REGION_H +#define LABCOLOR_ARB_REGION_H + +#include +#include +#include + +#include "interval.h" + +typedef enum { + LC_REGION_INSIDE = 0, + LC_REGION_OUTSIDE = 1, + LC_REGION_BOUNDARY_UNPROVEN = 2, + LC_REGION_RESOURCE_LIMIT_REACHED = 3 +} lc_region_outcome; + +typedef struct { + arb_struct tone; + arb_struct center_a; + arb_struct center_b; + arb_struct radius_squared; +} lc_region_knot; + +typedef struct { + arb_struct metric_aa; + arb_struct metric_ab; + arb_struct metric_bb; + lc_region_knot *knots; + size_t knot_count; +} lc_region; + +typedef struct { + lc_region_outcome outcome; + lc_status formula_status; + bool exact_boundary; + bool has_enclosure; + uint64_t exact_branch; + uint64_t consumed_branches; + arb_struct enclosure; +} lc_region_result; + +bool lc_region_init(lc_region *region, size_t knot_count); +void lc_region_clear(lc_region *region); +void lc_region_result_init(lc_region_result *result); +void lc_region_result_clear(lc_region_result *result); +void lc_region_decide( + lc_region_result *result, + arb_srcptr point, + const lc_region *region, + slong precision, + uint64_t branch_grant +); +void lc_region_evaluate_rgb( + lc_region_result *result, + const uint8_t rgb[3], + arb_srcptr context, + uint8_t surround, + const lc_region *region, + slong precision, + uint64_t branch_grant +); + +#endif diff --git a/proof/region/v1/arb/evaluator/wire.c b/proof/region/v1/arb/evaluator/wire.c new file mode 100644 index 00000000..c9f7e4d6 --- /dev/null +++ b/proof/region/v1/arb/evaluator/wire.c @@ -0,0 +1,616 @@ +#include "wire.h" + +#include +#include +#include +#include + +#include "hash.h" + +typedef struct { + const uint8_t *bytes; + size_t length; + size_t offset; + lc_wire_error *error; +} reader; + +static const uint8_t job_magic[8] = {'L', 'C', 'J', 'O', 'B', '1', 0, 0}; +static const uint8_t domain_magic[8] = {'L', 'C', 'D', 'O', 'M', '1', 0, 0}; +static const uint8_t policy_magic[8] = {'L', 'C', 'P', 'O', 'L', '1', 0, 0}; +static const uint8_t definition_domain[] = "labcolors.contextual-region-family-provider.v1\0"; +static const uint8_t formula_domain[] = "labcolors.nominal-exact-real-lift.ascii-ssa.v1\0"; +static const uint8_t domain_identity_label[] = "labcolors.proof-region.domain.v1\0"; +static const uint8_t policy_identity_label[] = "labcolors.proof-region.policy.v1\0"; +static const uint8_t job_identity_label[] = "labcolors.proof-region.job.v1\0"; +/* The registered V1 SSA has this exact wire length; changing either is a new + formula release, never a permissive parser adjustment. */ +static const size_t formula_spec_bytes_v1 = 24434; +static const uint8_t formula_release_v1[32] = { + 0x2c, 0x62, 0x6d, 0x8e, 0xe6, 0x0e, 0xeb, 0x62, + 0xae, 0x4d, 0xb5, 0x36, 0x60, 0xd6, 0x1b, 0xbc, + 0x25, 0xe0, 0xef, 0xd4, 0xe5, 0x57, 0xf0, 0xdc, + 0x1e, 0x77, 0x56, 0x5c, 0x13, 0x0b, 0x6e, 0x52, +}; + +static bool +reject(reader *input, lc_wire_error error) +{ + if (*input->error == LC_WIRE_OK) { + *input->error = error; + } + return false; +} + +static size_t +remaining(const reader *input) +{ + return input->length - input->offset; +} + +static bool +take(reader *input, size_t length, lc_slice *slice) +{ + if (length > remaining(input)) { + return reject(input, LC_WIRE_TRUNCATED); + } + slice->bytes = input->bytes + input->offset; + slice->length = length; + input->offset += length; + return true; +} + +static bool +expect(reader *input, const uint8_t *bytes, size_t length, lc_wire_error error) +{ + lc_slice actual; + + return take(input, length, &actual) + && (memcmp(actual.bytes, bytes, length) == 0 || reject(input, error)); +} + +static bool +read_u8(reader *input, uint8_t *value) +{ + lc_slice bytes; + + if (!take(input, 1, &bytes)) { + return false; + } + *value = bytes.bytes[0]; + return true; +} + +static bool +read_u32(reader *input, uint32_t *value) +{ + lc_slice bytes; + + if (!take(input, 4, &bytes)) { + return false; + } + *value = ((uint32_t) bytes.bytes[0] << 24) + | ((uint32_t) bytes.bytes[1] << 16) + | ((uint32_t) bytes.bytes[2] << 8) + | (uint32_t) bytes.bytes[3]; + return true; +} + +static bool +read_u64(reader *input, uint64_t *value) +{ + lc_slice bytes; + uint64_t result = 0; + + if (!take(input, 8, &bytes)) { + return false; + } + for (size_t index = 0; index < 8; ++index) { + result = (result << 8) | bytes.bytes[index]; + } + *value = result; + return true; +} + +static bool +read_blob(reader *input, size_t exact_length, lc_slice *value) +{ + uint64_t declared; + + if (!read_u64(input, &declared)) { + return false; + } + if (declared > SIZE_MAX || (exact_length != SIZE_MAX && declared != exact_length)) { + return reject(input, LC_WIRE_LENGTH_OUT_OF_BOUNDS); + } + if ((size_t) declared > remaining(input)) { + return reject(input, LC_WIRE_LENGTH_OUT_OF_BOUNDS); + } + return take(input, (size_t) declared, value); +} + +static bool +finish(reader *input) +{ + return remaining(input) == 0 || reject(input, LC_WIRE_TRAILING_BYTES); +} + +void +lc_write_u32_be(uint8_t output[4], uint32_t value) +{ + output[0] = (uint8_t) (value >> 24); + output[1] = (uint8_t) (value >> 16); + output[2] = (uint8_t) (value >> 8); + output[3] = (uint8_t) value; +} + +void +lc_write_u64_be(uint8_t output[8], uint64_t value) +{ + for (size_t index = 0; index < 8; ++index) { + output[7 - index] = (uint8_t) (value >> (index * 8)); + } +} + +static void +content_identity( + const uint8_t *label, + size_t label_length, + const uint8_t *bytes, + size_t length, + uint8_t digest[32] +) +{ + lc_sha256_context context; + uint8_t encoded_length[8]; + + lc_write_u64_be(encoded_length, (uint64_t) length); + lc_sha256_init(&context); + lc_sha256_update(&context, label, label_length); + lc_sha256_update(&context, encoded_length, sizeof(encoded_length)); + lc_sha256_update(&context, bytes, length); + lc_sha256_finish(&context, digest); +} + +static bool +exact_bits(lc_slice field, arb_t output) +{ + uint64_t bits = 0; + + if (field.length != 8) { + return false; + } + for (size_t index = 0; index < 8; ++index) { + bits = (bits << 8) | field.bytes[index]; + } + return lc_set_dyadic_bits(output, bits) == LC_OK; +} + +static bool +is_one_byte(lc_slice field, uint8_t value) +{ + return field.length == 1 && field.bytes[0] == value; +} + +static bool +parse_definition(lc_job *job, lc_slice encoded, reader *outer) +{ + static const size_t lengths[22] = { + sizeof(definition_domain) - 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 4, + 8, 8, 1, 1, 1, 32, 1, 8, 8, 8, 8, + }; + reader input = {encoded.bytes, encoded.length, 0, outer->error}; + lc_slice fields[22]; + uint64_t knot_count; + arb_t determinant; + arb_t product; + arb_t one; + + for (size_t index = 0; index < 22; ++index) { + if (!read_blob(&input, lengths[index], fields + index)) { + return false; + } + } + knot_count = 0; + for (size_t index = 0; index < 8; ++index) { + knot_count = (knot_count << 8) | fields[21].bytes[index]; + } + if (knot_count == 0 || knot_count > SIZE_MAX / 64 + || remaining(&input) != (size_t) knot_count * 64) { + return reject(&input, LC_WIRE_NONCANONICAL); + } + if (memcmp(fields[0].bytes, definition_domain, sizeof(definition_domain) - 1) != 0) { + return reject(&input, LC_WIRE_UNKNOWN_RELEASE); + } + for (size_t index = 1; index <= 17; ++index) { + bool fixed_one = index == 1 || index == 2 || index == 3 || index == 4 + || index == 5 || index == 6 || index == 8 || index == 9 + || index == 14 || index == 15 || index == 17; + if (fixed_one && !is_one_byte(fields[index], 1)) { + return reject(&input, LC_WIRE_UNKNOWN_RELEASE); + } + } + if (memcmp(fields[7].bytes, "\x01\x01\x01\x01", 4) != 0 + || memcmp(fields[10].bytes, "\x01\x01\x01\x01", 4) != 0 + || fields[13].bytes[0] < 1 || fields[13].bytes[0] > 3 + || memcmp(fields[16].bytes, formula_release_v1, 32) != 0) { + return reject(&input, LC_WIRE_UNKNOWN_RELEASE); + } + + arb_init(job->context + 0); + arb_init(job->context + 1); + job->context_ready = true; + if (!exact_bits(fields[11], job->context + 0) + || !exact_bits(fields[12], job->context + 1) + || !arb_is_positive(job->context + 0) + || !arb_is_positive(job->context + 1)) { + return reject(&input, LC_WIRE_NONCANONICAL); + } + arb_init(one); + arb_one(one); + if (!arb_le(job->context + 1, one)) { + arb_clear(one); + return reject(&input, LC_WIRE_NONCANONICAL); + } + arb_clear(one); + job->surround = fields[13].bytes[0]; + memcpy(job->formula_release, fields[16].bytes, 32); + + if (!lc_region_init(&job->region, (size_t) knot_count)) { + return reject(&input, LC_WIRE_ALLOCATION_FAILED); + } + job->region_ready = true; + if (!exact_bits(fields[18], &job->region.metric_aa) + || !exact_bits(fields[19], &job->region.metric_ab) + || !exact_bits(fields[20], &job->region.metric_bb) + || !arb_is_positive(&job->region.metric_aa)) { + return reject(&input, LC_WIRE_NONCANONICAL); + } + arb_init(determinant); + arb_init(product); + /* Binary64 coordinates are exact dyadics. Exact precision keeps SPD + admission independent of their exponent span. */ + arb_mul( + determinant, + &job->region.metric_aa, + &job->region.metric_bb, + ARF_PREC_EXACT + ); + arb_mul(product, &job->region.metric_ab, &job->region.metric_ab, ARF_PREC_EXACT); + arb_sub(determinant, determinant, product, ARF_PREC_EXACT); + if (!arb_is_exact(determinant) || !arb_is_positive(determinant)) { + arb_clear(product); + arb_clear(determinant); + return reject(&input, LC_WIRE_NONCANONICAL); + } + arb_clear(product); + arb_clear(determinant); + + for (size_t index = 0; index < (size_t) knot_count; ++index) { + lc_slice knot[4]; + lc_region_knot *target = job->region.knots + index; + + for (size_t coordinate = 0; coordinate < 4; ++coordinate) { + if (!read_blob(&input, 8, knot + coordinate)) { + return false; + } + } + if (!exact_bits(knot[0], &target->tone) + || !exact_bits(knot[1], &target->center_a) + || !exact_bits(knot[2], &target->center_b) + || !exact_bits(knot[3], &target->radius_squared) + || !arb_is_nonnegative(&target->radius_squared) + || (index != 0 && !arb_lt(&job->region.knots[index - 1].tone, &target->tone))) { + return reject(&input, LC_WIRE_NONCANONICAL); + } + } + return finish(&input); +} + +static bool +parse_domain(lc_domain *domain, lc_slice encoded, const uint8_t expected[32], reader *outer) +{ + reader input = {encoded.bytes, encoded.length, 0, outer->error}; + uint8_t release; + uint64_t range_count; + uint64_t maximum; + uint64_t total = 0; + + if (!expect(&input, domain_magic, sizeof(domain_magic), LC_WIRE_BAD_MAGIC) + || !read_u8(&input, &release) || release != 1 + || !read_u64(&input, &domain->point_count) + || domain->point_count == 0 || domain->point_count > UINT64_C(0x1000000) + || !read_u64(&input, &range_count)) { + return *input.error != LC_WIRE_OK + ? false + : reject(&input, LC_WIRE_NONCANONICAL); + } + maximum = domain->point_count; + if (UINT64_C(0x1000001) - domain->point_count < maximum) { + maximum = UINT64_C(0x1000001) - domain->point_count; + } + if (range_count == 0 || range_count > maximum || range_count > SIZE_MAX / sizeof(*domain->ranges) + || range_count > remaining(&input) / 8 || (size_t) range_count * 8 != remaining(&input)) { + return reject(&input, LC_WIRE_LENGTH_OUT_OF_BOUNDS); + } + domain->ranges = calloc((size_t) range_count, sizeof(*domain->ranges)); + if (domain->ranges == NULL) { + return reject(&input, LC_WIRE_ALLOCATION_FAILED); + } + domain->range_count = (size_t) range_count; + for (size_t index = 0; index < domain->range_count; ++index) { + lc_ordinal_range *range = domain->ranges + index; + + if (!read_u32(&input, &range->start) || !read_u32(&input, &range->end)) { + return false; + } + if (range->start >= range->end || range->end > UINT32_C(0x1000000) + || (index != 0 && range->start <= domain->ranges[index - 1].end)) { + return reject(&input, LC_WIRE_NONCANONICAL); + } + total += (uint64_t) range->end - range->start; + } + if (total != domain->point_count || !finish(&input)) { + return *input.error != LC_WIRE_OK + ? false + : reject(&input, LC_WIRE_NONCANONICAL); + } + content_identity( + domain_identity_label, + sizeof(domain_identity_label) - 1, + encoded.bytes, + encoded.length, + domain->identity + ); + return memcmp(domain->identity, expected, 32) == 0 + || reject(&input, LC_WIRE_DIGEST_MISMATCH); +} + +static bool +parse_policy(lc_arb_policy *policy, lc_slice encoded, const uint8_t expected[32], reader *outer) +{ + reader input = {encoded.bytes, encoded.length, 0, outer->error}; + uint8_t equality_release; + uint8_t comparator_count; + + if (!expect(&input, policy_magic, sizeof(policy_magic), LC_WIRE_BAD_MAGIC) + || !read_u8(&input, &equality_release) + || !read_u8(&input, &comparator_count)) { + return false; + } + if (equality_release != 1 || comparator_count != 2) { + return reject(&input, LC_WIRE_UNKNOWN_RELEASE); + } + for (uint8_t expected_kind = 1; expected_kind <= 2; ++expected_kind) { + uint8_t kind; + uint32_t rung_count; + uint32_t previous = 0; + size_t minimum_tail; + + if (!read_u8(&input, &kind) || !read_u32(&input, &rung_count)) { + return false; + } + minimum_tail = expected_kind == 1 ? 41 : 16; + if (kind != expected_kind || rung_count == 0 || remaining(&input) < minimum_tail + || rung_count > (remaining(&input) - minimum_tail) / 4) { + return reject(&input, LC_WIRE_NONCANONICAL); + } + if (expected_kind == 1) { + if ((size_t) rung_count > SIZE_MAX / sizeof(*policy->precision_ladder)) { + return reject(&input, LC_WIRE_LENGTH_OUT_OF_BOUNDS); + } + policy->precision_ladder = calloc(rung_count, sizeof(*policy->precision_ladder)); + if (policy->precision_ladder == NULL) { + return reject(&input, LC_WIRE_ALLOCATION_FAILED); + } + policy->precision_count = rung_count; + } + for (size_t index = 0; index < rung_count; ++index) { + uint32_t precision; + + if (!read_u32(&input, &precision)) { + return false; + } + if (precision == 0 || (index != 0 && precision <= previous)) { + return reject(&input, LC_WIRE_NONCANONICAL); + } + if (expected_kind == 1) { + policy->precision_ladder[index] = precision; + } + previous = precision; + } + if (expected_kind == 1) { + if (!read_u64(&input, &policy->per_point_work) + || !read_u64(&input, &policy->global_pregrant)) { + return false; + } + } else { + uint64_t ignored; + + if (!read_u64(&input, &ignored) || !read_u64(&input, &ignored)) { + return false; + } + } + } + if (!finish(&input)) { + return false; + } + content_identity( + policy_identity_label, + sizeof(policy_identity_label) - 1, + encoded.bytes, + encoded.length, + policy->identity + ); + return memcmp(policy->identity, expected, 32) == 0 + || reject(&input, LC_WIRE_DIGEST_MISMATCH); +} + +static bool +formula_release(lc_slice formula, uint8_t digest[32]) +{ + lc_sha256_context context; + uint8_t length[8]; + + lc_write_u64_be(length, (uint64_t) formula.length); + lc_sha256_init(&context); + lc_sha256_update(&context, formula_domain, sizeof(formula_domain) - 1); + lc_sha256_update(&context, length, sizeof(length)); + lc_sha256_update(&context, formula.bytes, formula.length); + lc_sha256_finish(&context, digest); + return memcmp(digest, formula_release_v1, 32) == 0; +} + +bool +lc_parse_job( + lc_job *job, + const uint8_t *bytes, + size_t length, + lc_wire_error *error +) +{ + reader input; + lc_slice definition; + lc_slice formula; + lc_slice domain; + lc_slice policy; + lc_slice definition_digest; + lc_slice declared_formula_release; + lc_slice domain_identity; + lc_slice policy_identity; + uint8_t actual[32]; + + memset(job, 0, sizeof(*job)); + *error = LC_WIRE_OK; + input = (reader) {bytes, length, 0, error}; + if (!expect(&input, job_magic, sizeof(job_magic), LC_WIRE_BAD_MAGIC) + || !take(&input, 32, &definition_digest) + || !read_blob(&input, SIZE_MAX, &definition) + || !take(&input, 32, &declared_formula_release) + || !read_blob(&input, formula_spec_bytes_v1, &formula) + || !take(&input, 32, &domain_identity) + || !read_blob(&input, SIZE_MAX, &domain) + || !take(&input, 32, &policy_identity) + || !read_blob(&input, SIZE_MAX, &policy) + || !finish(&input)) { + lc_job_clear(job); + return false; + } + lc_sha256(definition.bytes, definition.length, actual); + if (memcmp(actual, definition_digest.bytes, 32) != 0) { + *error = LC_WIRE_DIGEST_MISMATCH; + lc_job_clear(job); + return false; + } + if (!parse_definition(job, definition, &input) + || memcmp(declared_formula_release.bytes, job->formula_release, 32) != 0 + || !formula_release(formula, actual) + || memcmp(actual, declared_formula_release.bytes, 32) != 0 + || !parse_domain(&job->domain, domain, domain_identity.bytes, &input) + || !parse_policy(&job->policy, policy, policy_identity.bytes, &input)) { + if (*error == LC_WIRE_OK) { + *error = LC_WIRE_DIGEST_MISMATCH; + } + lc_job_clear(job); + return false; + } + content_identity( + job_identity_label, + sizeof(job_identity_label) - 1, + bytes, + length, + job->job_identity + ); + return true; +} + +void +lc_job_clear(lc_job *job) +{ + free(job->policy.precision_ladder); + free(job->domain.ranges); + if (job->region_ready) { + lc_region_clear(&job->region); + } + if (job->context_ready) { + arb_clear(job->context + 1); + arb_clear(job->context + 0); + } + memset(job, 0, sizeof(*job)); +} + +void +lc_domain_iterator_init(lc_domain_iterator *iterator, const lc_domain *domain) +{ + iterator->domain = domain; + iterator->range_index = 0; + iterator->ordinal = domain->ranges[0].start; + iterator->emitted = 0; +} + +bool +lc_domain_iterator_next(lc_domain_iterator *iterator, uint32_t *ordinal) +{ + if (iterator->emitted == iterator->domain->point_count) { + return false; + } + *ordinal = iterator->ordinal; + ++iterator->emitted; + ++iterator->ordinal; + if (iterator->ordinal == iterator->domain->ranges[iterator->range_index].end + && iterator->emitted != iterator->domain->point_count) { + /* Canonical parsing proves ordered disjoint ranges whose sizes sum to + point_count, so remaining output implies that a next range exists. */ + ++iterator->range_index; + iterator->ordinal = iterator->domain->ranges[iterator->range_index].start; + } + return true; +} + +void +lc_ordinal_to_rgb(uint32_t ordinal, uint8_t rgb[3]) +{ + rgb[0] = (uint8_t) (ordinal >> 16); + rgb[1] = (uint8_t) (ordinal >> 8); + rgb[2] = (uint8_t) ordinal; +} + +const char * +lc_wire_error_name(lc_wire_error error) +{ + static const char *const names[] = { + "ok", + "truncated", + "trailing_bytes", + "length_out_of_bounds", + "bad_magic", + "unknown_release", + "noncanonical", + "digest_mismatch", + "allocation_failed", + }; + + return (unsigned) error < sizeof(names) / sizeof(names[0]) + ? names[error] + : "unknown_wire_error"; +} + +bool +lc_write_all(int descriptor, const uint8_t *bytes, size_t length) +{ + while (length != 0) { + ssize_t written = write(descriptor, bytes, length); + + if (written < 0) { + if (errno == EINTR) { + continue; + } + return false; + } + if (written == 0) { + return false; + } + bytes += (size_t) written; + length -= (size_t) written; + } + return true; +} diff --git a/proof/region/v1/arb/evaluator/wire.h b/proof/region/v1/arb/evaluator/wire.h new file mode 100644 index 00000000..bd757fe4 --- /dev/null +++ b/proof/region/v1/arb/evaluator/wire.h @@ -0,0 +1,82 @@ +#ifndef LABCOLOR_ARB_WIRE_H +#define LABCOLOR_ARB_WIRE_H + +#include +#include +#include + +#include "region.h" + +typedef enum { + LC_WIRE_OK = 0, + LC_WIRE_TRUNCATED = 1, + LC_WIRE_TRAILING_BYTES = 2, + LC_WIRE_LENGTH_OUT_OF_BOUNDS = 3, + LC_WIRE_BAD_MAGIC = 4, + LC_WIRE_UNKNOWN_RELEASE = 5, + LC_WIRE_NONCANONICAL = 6, + LC_WIRE_DIGEST_MISMATCH = 7, + LC_WIRE_ALLOCATION_FAILED = 8 +} lc_wire_error; + +typedef struct { + const uint8_t *bytes; + size_t length; +} lc_slice; + +typedef struct { + uint32_t start; + uint32_t end; +} lc_ordinal_range; + +typedef struct { + lc_ordinal_range *ranges; + size_t range_count; + uint64_t point_count; + uint8_t identity[32]; +} lc_domain; + +typedef struct { + uint32_t *precision_ladder; + size_t precision_count; + uint64_t per_point_work; + uint64_t global_pregrant; + uint8_t identity[32]; +} lc_arb_policy; + +typedef struct { + lc_region region; + arb_struct context[2]; + uint8_t surround; + lc_domain domain; + lc_arb_policy policy; + uint8_t formula_release[32]; + uint8_t job_identity[32]; + bool context_ready; + bool region_ready; +} lc_job; + +typedef struct { + const lc_domain *domain; + size_t range_index; + uint32_t ordinal; + uint64_t emitted; +} lc_domain_iterator; + +bool lc_parse_job( + lc_job *job, + const uint8_t *bytes, + size_t length, + lc_wire_error *error +); +void lc_job_clear(lc_job *job); +void lc_domain_iterator_init(lc_domain_iterator *iterator, const lc_domain *domain); +bool lc_domain_iterator_next(lc_domain_iterator *iterator, uint32_t *ordinal); +void lc_ordinal_to_rgb(uint32_t ordinal, uint8_t rgb[3]); +const char *lc_wire_error_name(lc_wire_error error); + +bool lc_write_all(int descriptor, const uint8_t *bytes, size_t length); +void lc_write_u32_be(uint8_t output[4], uint32_t value); +void lc_write_u64_be(uint8_t output[8], uint64_t value); + +#endif diff --git a/proof/region/v1/arb/executor.py b/proof/region/v1/arb/executor.py new file mode 100644 index 00000000..9d68b605 --- /dev/null +++ b/proof/region/v1/arb/executor.py @@ -0,0 +1,2266 @@ +#!/usr/bin/env python3 +"""Fail-closed Linux process boundary for the Arb evaluator. + +This module returns process observations only. A caller must bind those +observations to source/build evidence elsewhere; no value here can certify that +provenance relationship. +""" + +from __future__ import annotations + +import ctypes +import errno as errno_module +import fcntl +import hashlib +import itertools +import os +import platform +import posixpath +import resource +import selectors +import signal +import struct +import sys +import threading +import time +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Callable, NoReturn, Protocol, TypeAlias + + +SANDBOX_POLICY_RELEASE_V1 = "labcolors.arb.executor.linux-x86_64.v1" + +# Linux UAPI values are fixed by fcntl.h. Requiring F_SEAL_EXEC makes an older +# kernel an explicit Unsupported host instead of silently weakening the object. +F_SEAL_SEAL_V1 = 0x0001 +F_SEAL_SHRINK_V1 = 0x0002 +F_SEAL_GROW_V1 = 0x0004 +F_SEAL_WRITE_V1 = 0x0008 +F_SEAL_EXEC_V1 = 0x0020 +REQUIRED_FILE_SEALS_V1 = ( + F_SEAL_SEAL_V1 + | F_SEAL_SHRINK_V1 + | F_SEAL_GROW_V1 + | F_SEAL_WRITE_V1 + | F_SEAL_EXEC_V1 +) + +_MFD_CLOEXEC = 0x0001 +_MFD_ALLOW_SEALING = 0x0002 +_MFD_EXEC = 0x0010 +_F_ADD_SEALS = 1033 +_F_GET_SEALS = 1034 +_AT_EMPTY_PATH = 0x1000 + +_SYS_SECCOMP_X86_64 = 317 +_SYS_EXECVEAT_X86_64 = 322 +_SYS_CLOSE_RANGE_X86_64 = 436 +_SYS_PRLIMIT64_X86_64 = 302 + +_CLONE_NEWNS = 0x00020000 +_CLONE_NEWUSER = 0x10000000 +_CLONE_NEWNET = 0x40000000 +_MS_REC = 0x4000 +_MS_PRIVATE = 1 << 18 + +_PR_SET_NO_NEW_PRIVS = 38 +_SECCOMP_SET_MODE_FILTER = 1 +_SECCOMP_FILTER_FLAG_TSYNC = 1 +_SECCOMP_RET_KILL_PROCESS = 0x80000000 +_SECCOMP_RET_ALLOW = 0x7FFF0000 +_AUDIT_ARCH_X86_64 = 0xC000003E + +_BPF_LD_W_ABS = 0x20 +_BPF_JMP_JEQ_K = 0x15 +_BPF_RET_K = 0x06 + +_CHILD_PACKET = struct.Struct(">4sBBI") +_CHILD_PACKET_MAGIC = b"LCXE" +_ELF_HEADER = struct.Struct("<16sHHIQQQIHHHHHH") +_ELF_PROGRAM_HEADER = struct.Struct(" None: + super().__init__(f"{field}: {reason.value}") + self.reason = reason + self.field = field + + +class CapabilityReasonV1(str, Enum): + HOST_NOT_LINUX = "host_not_linux" + ARCHITECTURE_NOT_SUPPORTED = "architecture_not_supported" + CGROUP_PARENT_NOT_DECLARED = "cgroup_parent_not_declared" + CGROUP_V2_UNAVAILABLE = "cgroup_v2_unavailable" + EXECUTABLE_MEMFD_UNAVAILABLE = "executable_memfd_unavailable" + FILE_SEALS_UNAVAILABLE = "file_seals_unavailable" + EXECVEAT_UNAVAILABLE = "execveat_unavailable" + CLOSE_RANGE_UNAVAILABLE = "close_range_unavailable" + NETWORK_NAMESPACE_UNAVAILABLE = "network_namespace_unavailable" + SECCOMP_FILTER_UNAVAILABLE = "seccomp_filter_unavailable" + STANDARD_FDS_UNAVAILABLE = "standard_fds_unavailable" + OBSERVER_NOT_SINGLE_THREADED = "observer_not_single_threaded" + OBSERVER_TASK_BUDGET_UNAVAILABLE = "observer_task_budget_unavailable" + OBSERVATION_INVALIDATED = "observation_invalidated" + KERNEL_API_UNAVAILABLE = "kernel_api_unavailable" + + +@dataclass(frozen=True) +class CapabilityFailureV1: + reason: CapabilityReasonV1 + errno: int | None + + def __post_init__(self) -> None: + if type(self.reason) is not CapabilityReasonV1: + raise TypeError("reason must be CapabilityReasonV1") + if self.errno is not None and (type(self.errno) is not int or self.errno <= 0): + raise TypeError("errno must be a positive int or None") + + +@dataclass(frozen=True) +class UnsupportedV1: + failures: tuple[CapabilityFailureV1, ...] + + def __post_init__(self) -> None: + if ( + type(self.failures) is not tuple + or not self.failures + or any(type(item) is not CapabilityFailureV1 for item in self.failures) + or len(set(self.failures)) != len(self.failures) + ): + raise TypeError("failures must be a nonempty unique tuple") + + +@dataclass(frozen=True) +class SupportedV1: + platform: str + sandbox_policy_release: str + + def __post_init__(self) -> None: + if type(self.platform) is not str or not self.platform: + raise TypeError("platform must be a nonempty str") + if self.sandbox_policy_release != SANDBOX_POLICY_RELEASE_V1: + raise TypeError("unknown sandbox policy release") + + +CapabilityReportV1: TypeAlias = SupportedV1 | UnsupportedV1 + + +def _invalidated_capability_report_v1() -> UnsupportedV1: + return UnsupportedV1( + ( + CapabilityFailureV1( + CapabilityReasonV1.OBSERVATION_INVALIDATED, + errno_module.EBUSY, + ), + ) + ) + + +@dataclass(frozen=True) +class ExecutionLimitsV1: + max_executable_bytes: int + max_stdin_bytes: int + max_argument_bytes: int + max_stdout_bytes: int + max_stderr_bytes: int + wall_timeout_ns: int + memory_max_bytes: int + pids_max: int + + def __post_init__(self) -> None: + positive = ( + "max_executable_bytes", + "max_stdin_bytes", + "max_argument_bytes", + "wall_timeout_ns", + "memory_max_bytes", + "pids_max", + ) + nonnegative = ("max_stdout_bytes", "max_stderr_bytes") + for field_name in positive: + value = getattr(self, field_name) + if type(value) is not int or value <= 0: + raise ExecutionRequestErrorV1(RequestReasonV1.INVALID_LIMIT, field_name) + for field_name in nonnegative: + value = getattr(self, field_name) + if type(value) is not int or value < 0: + raise ExecutionRequestErrorV1(RequestReasonV1.INVALID_LIMIT, field_name) + # V1's syscall policy denies clone/fork/vfork; a larger cgroup task + # budget would advertise a concurrency capability the executor lacks. + if self.pids_max != 1: + raise ExecutionRequestErrorV1(RequestReasonV1.INVALID_LIMIT, "pids_max") + + +@dataclass(frozen=True) +class ExecutionRequestV1: + executable: bytes + argv: tuple[bytes, ...] + environment: tuple[tuple[bytes, bytes], ...] + cwd: bytes + stdin: bytes + umask: int + limits: ExecutionLimitsV1 + + def __post_init__(self) -> None: + if type(self.limits) is not ExecutionLimitsV1: + _request_fail(RequestReasonV1.WRONG_TYPE, "limits") + if type(self.executable) is not bytes: + _request_fail(RequestReasonV1.WRONG_TYPE, "executable") + if not self.executable or len(self.executable) > self.limits.max_executable_bytes: + _request_fail(RequestReasonV1.LIMIT_EXCEEDED, "executable") + require_static_x86_64_elf_v1(self.executable) + + if type(self.argv) is not tuple or not self.argv: + _request_fail(RequestReasonV1.WRONG_TYPE, "argv") + for index, item in enumerate(self.argv): + _require_bytes_without_nul(item, f"argv[{index}]") + if not self.argv[0]: + _request_fail(RequestReasonV1.EMPTY_ARGV_ZERO, "argv[0]") + + if type(self.environment) is not tuple: + _request_fail(RequestReasonV1.WRONG_TYPE, "environment") + previous: bytes | None = None + argument_bytes = sum(len(item) + 1 for item in self.argv) + for index, item in enumerate(self.environment): + if type(item) is not tuple or len(item) != 2: + _request_fail(RequestReasonV1.WRONG_TYPE, f"environment[{index}]") + key, value = item + _require_bytes_without_nul(key, f"environment[{index}].key") + _require_bytes_without_nul(value, f"environment[{index}].value") + if not key or b"=" in key: + _request_fail( + RequestReasonV1.INVALID_ENVIRONMENT_KEY, + f"environment[{index}].key", + ) + if previous == key: + _request_fail(RequestReasonV1.DUPLICATE_ENVIRONMENT, "environment") + if previous is not None and previous > key: + _request_fail(RequestReasonV1.NONCANONICAL_ENVIRONMENT, "environment") + previous = key + argument_bytes += len(key) + len(value) + 2 + if argument_bytes > self.limits.max_argument_bytes: + _request_fail(RequestReasonV1.LIMIT_EXCEEDED, "argv+environment") + + _require_bytes_without_nul(self.cwd, "cwd") + if not self.cwd.startswith(b"/"): + _request_fail(RequestReasonV1.RELATIVE_CWD, "cwd") + if ( + posixpath.normpath(self.cwd) != self.cwd + or self.cwd.startswith(b"//") + or (self.cwd != b"/" and self.cwd.endswith(b"/")) + ): + _request_fail(RequestReasonV1.NONCANONICAL_CWD, "cwd") + + if type(self.stdin) is not bytes: + _request_fail(RequestReasonV1.WRONG_TYPE, "stdin") + if len(self.stdin) > self.limits.max_stdin_bytes: + _request_fail(RequestReasonV1.LIMIT_EXCEEDED, "stdin") + if type(self.umask) is not int or not 0 <= self.umask <= 0o777: + _request_fail(RequestReasonV1.INVALID_LIMIT, "umask") + + +def _request_fail(reason: RequestReasonV1, field: str) -> NoReturn: + raise ExecutionRequestErrorV1(reason, field) + + +def _require_bytes_without_nul(value: object, field: str) -> None: + if type(value) is not bytes: + _request_fail(RequestReasonV1.WRONG_TYPE, field) + if b"\0" in value: + _request_fail(RequestReasonV1.NUL_BYTE, field) + + +def require_static_x86_64_elf_v1(data: bytes) -> None: + if type(data) is not bytes: + _request_fail(RequestReasonV1.WRONG_TYPE, "executable") + if len(data) < _ELF_HEADER.size: + _request_fail(RequestReasonV1.INVALID_ELF, "executable") + try: + ( + ident, + elf_type, + machine, + version, + _entry, + program_offset, + _section_offset, + _flags, + header_size, + program_entry_size, + program_count, + _section_entry_size, + _section_count, + _section_names, + ) = _ELF_HEADER.unpack_from(data) + except struct.error: + _request_fail(RequestReasonV1.INVALID_ELF, "executable") + if ( + ident[:7] != b"\x7fELF\x02\x01\x01" + or ident[7] not in (0, 3) + or elf_type not in (2, 3) + or machine != 62 + or version != 1 + or header_size != _ELF_HEADER.size + or program_entry_size != _ELF_PROGRAM_HEADER.size + or program_count == 0 + or program_offset < header_size + ): + _request_fail(RequestReasonV1.INVALID_ELF, "executable") + table_end = program_offset + program_count * program_entry_size + if table_end > len(data): + _request_fail(RequestReasonV1.INVALID_ELF, "executable") + + saw_load = False + dynamic_ranges: list[tuple[int, int]] = [] + for index in range(program_count): + offset = program_offset + index * program_entry_size + try: + ( + segment_type, + _segment_flags, + file_offset, + _virtual_address, + _physical_address, + file_size, + memory_size, + _alignment, + ) = _ELF_PROGRAM_HEADER.unpack_from(data, offset) + except struct.error: + _request_fail(RequestReasonV1.INVALID_ELF, "executable") + if file_size > memory_size or file_offset + file_size > len(data): + _request_fail(RequestReasonV1.INVALID_ELF, "executable") + if segment_type == 1: + saw_load = True + elif segment_type == 3: + _request_fail(RequestReasonV1.DYNAMIC_EXECUTABLE, "executable") + elif segment_type == 2: + dynamic_ranges.append((file_offset, file_size)) + if not saw_load: + _request_fail(RequestReasonV1.INVALID_ELF, "executable") + + for start, size in dynamic_ranges: + if size % _ELF_DYNAMIC_ENTRY.size != 0: + _request_fail(RequestReasonV1.INVALID_ELF, "executable") + saw_terminator = False + for offset in range(start, start + size, _ELF_DYNAMIC_ENTRY.size): + tag, _value = _ELF_DYNAMIC_ENTRY.unpack_from(data, offset) + if tag == 0: + saw_terminator = True + break + if tag == 1: + _request_fail(RequestReasonV1.DYNAMIC_EXECUTABLE, "executable") + if not saw_terminator: + _request_fail(RequestReasonV1.INVALID_ELF, "executable") + + +class OutputStreamV1(str, Enum): + STDOUT = "stdout" + STDERR = "stderr" + + +class SetupStageV1(int, Enum): + SEALED_EXECUTABLE = 1 + CGROUP_CREATE = 2 + CGROUP_ATTACH = 3 + CWD = 4 + NAMESPACE = 5 + MOUNT_PROPAGATION = 6 + FILE_DESCRIPTORS = 7 + SIGNAL_STATE = 8 + NO_NEW_PRIVILEGES = 9 + SECCOMP = 10 + EXECVEAT = 11 + OBSERVER_PRECONDITION = 12 + + +class ObserverReasonV1(str, Enum): + PROBE_FAILED = "probe_failed" + BACKEND_EXCEPTION = "backend_exception" + BACKEND_CONTRACT = "backend_contract" + CHILD_PROTOCOL = "child_protocol" + CGROUP_OBSERVATION = "cgroup_observation" + CLEANUP_FAILED = "cleanup_failed" + + +@dataclass(frozen=True) +class CompletedV1: + binary_sha256: bytes + stdout: bytes + stderr: bytes + + +@dataclass(frozen=True) +class ExitNonZeroV1: + binary_sha256: bytes + stdout: bytes + stderr: bytes + exit_code: int + + +@dataclass(frozen=True) +class SignaledV1: + binary_sha256: bytes + stdout: bytes + stderr: bytes + signal_number: int + core_dumped: bool + + +@dataclass(frozen=True) +class TimedOutV1: + binary_sha256: bytes + stdout: bytes + stderr: bytes + deadline_ns: int + + +@dataclass(frozen=True) +class OomKilledV1: + binary_sha256: bytes + stdout: bytes + stderr: bytes + oom_kill_delta: int + + +@dataclass(frozen=True) +class OutputLimitExceededV1: + binary_sha256: bytes + stdout: bytes + stderr: bytes + stream: OutputStreamV1 + limit: int + + +@dataclass(frozen=True) +class SandboxSetupFailedV1: + binary_sha256: bytes | None + stdout: bytes + stderr: bytes + stage: SetupStageV1 + errno: int + + +@dataclass(frozen=True) +class ResidualProcessesV1: + binary_sha256: bytes + stdout: bytes + stderr: bytes + + +@dataclass(frozen=True) +class ObserverFailureV1: + reason: ObserverReasonV1 + + +ExecutionResultV1: TypeAlias = ( + CompletedV1 + | ExitNonZeroV1 + | SignaledV1 + | TimedOutV1 + | OomKilledV1 + | OutputLimitExceededV1 + | SandboxSetupFailedV1 + | ResidualProcessesV1 + | ObserverFailureV1 + | UnsupportedV1 +) + + +@dataclass(frozen=True) +class _ProbeGuardV1: + """One controller-owned lease; a backend may observe but never renew it.""" + + _is_current: Callable[[], bool] + + def is_current(self) -> bool: + try: + return self._is_current() + except Exception: + return False + + +class ExecutionBackendV1(Protocol): + def probe(self, guard: _ProbeGuardV1) -> CapabilityReportV1: ... + + def run( + self, + request: ExecutionRequestV1, + capability: SupportedV1, + ) -> ExecutionResultV1: ... + + +class ControlledExecutorV1: + def __init__(self, backend: ExecutionBackendV1 | None = None) -> None: + self._backend = backend if backend is not None else NativeLinuxBackendV1() + # A fork snapshots Python locks and object identity, so an inherited + # controller cannot share the creator process's one-shot authority. + self._owner_pid = os.getpid() + self._capability_lock = threading.Lock() + self._capability_generation = 0 + self._capability_conflict_generation = 0 + self._active_capability_probes = 0 + self._issued_capability: SupportedV1 | None = None + self._issued_backend: ExecutionBackendV1 | None = None + + def _probe_is_current_v1( + self, + generation: int, + conflict_generation: int, + backend: ExecutionBackendV1, + ) -> bool: + with self._capability_lock: + return ( + os.getpid() == self._owner_pid + and generation == self._capability_generation + and conflict_generation == self._capability_conflict_generation + and backend is self._backend + ) + + def probe(self) -> CapabilityReportV1: + if os.getpid() != self._owner_pid: + return _invalidated_capability_report_v1() + with self._capability_lock: + self._capability_generation += 1 + generation = self._capability_generation + if self._active_capability_probes != 0: + self._capability_conflict_generation += 1 + self._issued_capability = None + self._issued_backend = None + return _invalidated_capability_report_v1() + conflict_generation = self._capability_conflict_generation + self._active_capability_probes += 1 + self._issued_capability = None + self._issued_backend = None + backend = self._backend + guard = _ProbeGuardV1( + lambda: self._probe_is_current_v1( + generation, + conflict_generation, + backend, + ) + ) + try: + report = backend.probe(guard) + except Exception: + report = UnsupportedV1( + ( + CapabilityFailureV1( + CapabilityReasonV1.KERNEL_API_UNAVAILABLE, + None, + ), + ) + ) + except BaseException: + with self._capability_lock: + self._active_capability_probes -= 1 + self._capability_conflict_generation += 1 + self._issued_capability = None + self._issued_backend = None + raise + if type(report) not in (SupportedV1, UnsupportedV1): + report = UnsupportedV1( + ( + CapabilityFailureV1( + CapabilityReasonV1.KERNEL_API_UNAVAILABLE, + None, + ), + ) + ) + with self._capability_lock: + self._active_capability_probes -= 1 + invalidated = ( + generation != self._capability_generation + or conflict_generation != self._capability_conflict_generation + or backend is not self._backend + ) + if invalidated: + self._issued_capability = None + self._issued_backend = None + return _invalidated_capability_report_v1() + if type(report) is SupportedV1: + # The backend reports host facts; it cannot mint authority. + # A fresh controller-owned object binds this exact successful + # probe generation, even when a backend reuses its report. + issued = SupportedV1( + report.platform, + report.sandbox_policy_release, + ) + self._issued_capability = issued + self._issued_backend = backend + return issued + return report + + def execute( + self, + request: ExecutionRequestV1, + capability: SupportedV1 | None = None, + ) -> ExecutionResultV1: + if os.getpid() != self._owner_pid: + return ObserverFailureV1(ObserverReasonV1.PROBE_FAILED) + if type(request) is not ExecutionRequestV1: + raise ExecutionRequestErrorV1(RequestReasonV1.WRONG_TYPE, "request") + if capability is None: + report = self.probe() + if type(report) is UnsupportedV1: + return report + capability = report + if type(capability) is not SupportedV1: + return ObserverFailureV1(ObserverReasonV1.PROBE_FAILED) + with self._capability_lock: + backend = self._issued_backend + if ( + capability is not self._issued_capability + or backend is None + or backend is not self._backend + ): + return ObserverFailureV1(ObserverReasonV1.PROBE_FAILED) + # The controller is the sole owner. Consumption precedes every + # backend operation, so retries and backend replacement fail shut. + self._issued_capability = None + self._issued_backend = None + try: + result = backend.run(request, capability) + except Exception: + return ObserverFailureV1(ObserverReasonV1.BACKEND_EXCEPTION) + if not result_matches_request_v1(result, request): + return ObserverFailureV1(ObserverReasonV1.BACKEND_CONTRACT) + return result + + +def _unsupported_is_well_typed_v1(report: UnsupportedV1) -> bool: + failures = report.failures + return ( + type(failures) is tuple + and bool(failures) + and all( + type(failure) is CapabilityFailureV1 + and type(failure.reason) is CapabilityReasonV1 + and ( + failure.errno is None + or (type(failure.errno) is int and failure.errno > 0) + ) + for failure in failures + ) + ) + + +def _result_matches_request_v1(result: object, request: ExecutionRequestV1) -> bool: + if type(request) is not ExecutionRequestV1: + return False + known = ( + CompletedV1, + ExitNonZeroV1, + SignaledV1, + TimedOutV1, + OomKilledV1, + OutputLimitExceededV1, + SandboxSetupFailedV1, + ResidualProcessesV1, + ObserverFailureV1, + UnsupportedV1, + ) + if type(result) not in known: + return False + if type(result) is ObserverFailureV1: + return type(result.reason) is ObserverReasonV1 + if type(result) is UnsupportedV1: + return _unsupported_is_well_typed_v1(result) + + stdout = result.stdout + stderr = result.stderr + if ( + type(stdout) is not bytes + or type(stderr) is not bytes + or len(stdout) > request.limits.max_stdout_bytes + or len(stderr) > request.limits.max_stderr_bytes + ): + return False + expected_digest = hashlib.sha256(request.executable).digest() + if type(result) is SandboxSetupFailedV1: + if result.binary_sha256 is not None and ( + type(result.binary_sha256) is not bytes + or len(result.binary_sha256) != len(expected_digest) + or result.binary_sha256 != expected_digest + ): + return False + return ( + type(result.stage) is SetupStageV1 + and type(result.errno) is int + and result.errno > 0 + ) + if ( + type(result.binary_sha256) is not bytes + or len(result.binary_sha256) != len(expected_digest) + or result.binary_sha256 != expected_digest + ): + return False + if type(result) is ExitNonZeroV1: + return type(result.exit_code) is int and result.exit_code > 0 + if type(result) is SignaledV1: + return ( + type(result.signal_number) is int + and result.signal_number > 0 + and type(result.core_dumped) is bool + ) + if type(result) is TimedOutV1: + return ( + type(result.deadline_ns) is int + and result.deadline_ns == request.limits.wall_timeout_ns + ) + if type(result) is OomKilledV1: + return type(result.oom_kill_delta) is int and result.oom_kill_delta > 0 + if type(result) is OutputLimitExceededV1: + expected_limit = ( + request.limits.max_stdout_bytes + if result.stream is OutputStreamV1.STDOUT + else request.limits.max_stderr_bytes + if result.stream is OutputStreamV1.STDERR + else None + ) + captured = ( + result.stdout + if result.stream is OutputStreamV1.STDOUT + else result.stderr + ) + return ( + expected_limit is not None + and type(result.limit) is int + and result.limit == expected_limit + and len(captured) == expected_limit + ) + return True + + +def result_matches_request_v1(result: object, request: ExecutionRequestV1) -> bool: + """Total validation for observations returned by an injected backend.""" + + try: + return _result_matches_request_v1(result, request) + except Exception: + return False + + +class _MemfdOperationsV1(Protocol): + def create_executable_memfd(self) -> int: ... + + def write_all(self, fd: int, data: bytes) -> None: ... + + def make_executable(self, fd: int) -> None: ... + + def add_seals(self, fd: int, seals: int) -> None: ... + + def get_seals(self, fd: int) -> int: ... + + def pread(self, fd: int, size: int, offset: int) -> bytes: ... + + def execveat( + self, + fd: int, + argv: tuple[bytes, ...], + environment: tuple[tuple[bytes, bytes], ...], + ) -> None: ... + + def close(self, fd: int) -> None: ... + + +@dataclass(frozen=True) +class _SealedExecutableV1: + fd: int + size: int + sha256: bytes + + def execveat( + self, + argv: tuple[bytes, ...], + environment: tuple[tuple[bytes, bytes], ...], + operations: _MemfdOperationsV1, + ) -> None: + operations.execveat(self.fd, argv, environment) + + +def _seal_executable_v1( + executable: bytes, + operations: _MemfdOperationsV1, +) -> _SealedExecutableV1: + fd = operations.create_executable_memfd() + try: + operations.write_all(fd, executable) + operations.make_executable(fd) + operations.add_seals(fd, REQUIRED_FILE_SEALS_V1) + actual_seals = operations.get_seals(fd) + if actual_seals & REQUIRED_FILE_SEALS_V1 != REQUIRED_FILE_SEALS_V1: + raise OSError(errno_module.ENOTSUP, "required file seals did not stick") + digest = hashlib.sha256() + offset = 0 + while offset < len(executable): + chunk = operations.pread(fd, min(1 << 20, len(executable) - offset), offset) + if not chunk: + raise OSError(errno_module.EIO, "sealed executable shortened while hashing") + digest.update(chunk) + offset += len(chunk) + if offset != len(executable): + raise OSError(errno_module.EIO, "sealed executable length changed") + return _SealedExecutableV1(fd, len(executable), digest.digest()) + except BaseException: + operations.close(fd) + raise + + +@dataclass(frozen=True) +class _ChildErrorV1: + stage: SetupStageV1 + errno: int + + +class ObserverProtocolErrorV1(ValueError): + pass + + +def _encode_child_error_packet_v1(stage: SetupStageV1, error_number: int) -> bytes: + if type(stage) is not SetupStageV1 or type(error_number) is not int or not 0 < error_number <= 0xFFFFFFFF: + raise ValueError("invalid child error") + return _CHILD_PACKET.pack(_CHILD_PACKET_MAGIC, 1, stage.value, error_number) + + +def _parse_child_error_packet_v1(packet: bytes) -> _ChildErrorV1: + if type(packet) is not bytes or len(packet) != _CHILD_PACKET.size: + raise ObserverProtocolErrorV1("noncanonical child packet length") + try: + magic, release, raw_stage, error_number = _CHILD_PACKET.unpack(packet) + stage = SetupStageV1(raw_stage) + except (struct.error, ValueError) as error: + raise ObserverProtocolErrorV1("invalid child packet") from error + if magic != _CHILD_PACKET_MAGIC or release != 1 or error_number == 0: + raise ObserverProtocolErrorV1("invalid child packet") + return _ChildErrorV1(stage, error_number) + + +class _SockFilter(ctypes.Structure): + _fields_ = ( + ("code", ctypes.c_ushort), + ("jt", ctypes.c_ubyte), + ("jf", ctypes.c_ubyte), + ("k", ctypes.c_uint32), + ) + + +class _SockFprog(ctypes.Structure): + _fields_ = ( + ("length", ctypes.c_ushort), + ("filters", ctypes.POINTER(_SockFilter)), + ) + + +class _NativeLinuxOperationsV1: + _runtime_syscalls = ( + 0, # read: only inherited stdin remains readable + 1, # write: only inherited stdout/stderr remain writable + 3, # close + 5, # fstat + 8, # lseek + 9, # mmap + 10, # mprotect + 11, # munmap + 12, # brk + 13, # rt_sigaction + 14, # rt_sigprocmask + 15, # rt_sigreturn + 25, # mremap + 28, # madvise + 60, # exit + 131, # sigaltstack + 158, # arch_prctl + 202, # futex + 218, # set_tid_address + 231, # exit_group + 273, # set_robust_list + 334, # rseq + ) + + def __init__(self) -> None: + self._libc = ctypes.CDLL(None, use_errno=True) + + def create_executable_memfd(self) -> int: + if not hasattr(os, "memfd_create"): + raise OSError(errno_module.ENOSYS, "memfd_create unavailable") + return os.memfd_create( + "labcolors-arb-evaluator", + _MFD_CLOEXEC | _MFD_ALLOW_SEALING | _MFD_EXEC, + ) + + def pipe_cloexec(self) -> tuple[int, int]: + return os.pipe2(os.O_CLOEXEC) + + def write_all(self, fd: int, data: bytes) -> None: + view = memoryview(data) + offset = 0 + while offset < len(view): + try: + written = os.write(fd, view[offset:]) + except InterruptedError: + continue + if written <= 0: + raise OSError(errno_module.EIO, "short memfd write") + offset += written + + def make_executable(self, fd: int) -> None: + os.fchmod(fd, 0o500) + + def add_seals(self, fd: int, seals: int) -> None: + fcntl.fcntl(fd, _F_ADD_SEALS, seals) + + def get_seals(self, fd: int) -> int: + return int(fcntl.fcntl(fd, _F_GET_SEALS)) + + def pread(self, fd: int, size: int, offset: int) -> bytes: + return os.pread(fd, size, offset) + + def close(self, fd: int) -> None: + os.close(fd) + + def execveat( + self, + fd: int, + argv: tuple[bytes, ...], + environment: tuple[tuple[bytes, bytes], ...], + ) -> None: + argv_array = (ctypes.c_char_p * (len(argv) + 1))(*argv, None) + environment_bytes = tuple(key + b"=" + value for key, value in environment) + environment_array = (ctypes.c_char_p * (len(environment_bytes) + 1))( + *environment_bytes, + None, + ) + ctypes.set_errno(0) + result = self._libc.syscall( + _SYS_EXECVEAT_X86_64, + fd, + ctypes.c_char_p(b""), + argv_array, + environment_array, + _AT_EMPTY_PATH, + ) + error_number = ctypes.get_errno() + if result == -1: + raise OSError(error_number or errno_module.EIO, "execveat failed") + raise OSError(errno_module.EIO, "execveat unexpectedly returned") + + def probe_execveat(self) -> None: + ctypes.set_errno(0) + result = self._libc.syscall( + _SYS_EXECVEAT_X86_64, + -1, + ctypes.c_char_p(b""), + ctypes.c_void_p(), + ctypes.c_void_p(), + _AT_EMPTY_PATH, + ) + error_number = ctypes.get_errno() + if result != -1 or error_number != errno_module.EBADF: + raise OSError(error_number or errno_module.ENOSYS, "execveat unavailable") + + def probe_single_threaded(self) -> None: + try: + task_count = len(os.listdir("/proc/self/task")) + except OSError as error: + raise OSError(error.errno or errno_module.EIO, "cannot inspect observer tasks") from error + if task_count != 1: + raise OSError(errno_module.EBUSY, "observer is not single-threaded") + + def probe_standard_fds(self) -> None: + for descriptor in (0, 1, 2): + os.fstat(descriptor) + + def close_range_after_setup(self) -> None: + ctypes.set_errno(0) + result = self._libc.syscall( + _SYS_CLOSE_RANGE_X86_64, + 5, + ctypes.c_uint(0xFFFFFFFF), + 0, + ) + if result == -1: + raise OSError(ctypes.get_errno() or errno_module.EIO, "close_range failed") + + def probe_close_range(self) -> None: + ctypes.set_errno(0) + result = self._libc.syscall( + _SYS_CLOSE_RANGE_X86_64, + ctypes.c_uint(0xFFFFFFFF), + ctypes.c_uint(0xFFFFFFFF), + 0, + ) + if result == -1: + raise OSError(ctypes.get_errno() or errno_module.ENOSYS, "close_range unavailable") + + def enter_namespaces(self) -> None: + flags = _CLONE_NEWUSER | _CLONE_NEWNET | _CLONE_NEWNS + ctypes.set_errno(0) + if self._libc.unshare(flags) == -1: + raise OSError(ctypes.get_errno() or errno_module.EIO, "unshare failed") + + def make_mounts_private(self) -> None: + ctypes.set_errno(0) + result = self._libc.mount( + ctypes.c_void_p(), + ctypes.c_char_p(b"/"), + ctypes.c_void_p(), + ctypes.c_ulong(_MS_REC | _MS_PRIVATE), + ctypes.c_void_p(), + ) + if result == -1: + raise OSError(ctypes.get_errno() or errno_module.EIO, "mount propagation failed") + + def probe_namespaces(self) -> None: + read_fd, write_fd = os.pipe2(os.O_CLOEXEC) + pid = os.fork() + if pid == 0: + os.close(read_fd) + error_number = 0 + try: + self.enter_namespaces() + self.make_mounts_private() + except OSError as error: + error_number = error.errno or errno_module.EIO + try: + os.write(write_fd, error_number.to_bytes(4, "big")) + finally: + os._exit(0 if error_number == 0 else 1) + os.close(write_fd) + try: + packet = _read_exact_fd(read_fd, 4) + finally: + os.close(read_fd) + _wait_exact_child(pid) + if len(packet) != 4: + raise OSError(errno_module.EIO, "namespace probe lost") + error_number = int.from_bytes(packet, "big") + if error_number: + raise OSError(error_number, "namespace probe failed") + + def set_no_new_privileges(self) -> None: + ctypes.set_errno(0) + if self._libc.prctl(_PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) == -1: + raise OSError(ctypes.get_errno() or errno_module.EIO, "no_new_privs failed") + + def set_not_dumpable(self) -> None: + ctypes.set_errno(0) + if self._libc.prctl(4, 0, 0, 0, 0) == -1: # PR_SET_DUMPABLE + raise OSError(ctypes.get_errno() or errno_module.EIO, "PR_SET_DUMPABLE failed") + + def install_seccomp(self, exec_fd: int, setup_error_fd: int) -> None: + instructions = self._seccomp_program(exec_fd, setup_error_fd) + array = (_SockFilter * len(instructions))(*instructions) + program = _SockFprog(len(instructions), array) + ctypes.set_errno(0) + result = self._libc.syscall( + _SYS_SECCOMP_X86_64, + _SECCOMP_SET_MODE_FILTER, + _SECCOMP_FILTER_FLAG_TSYNC, + ctypes.byref(program), + ) + if result == -1: + raise OSError(ctypes.get_errno() or errno_module.EIO, "seccomp failed") + + def probe_seccomp(self) -> None: + pid = os.fork() + if pid == 0: + try: + self.set_no_new_privileges() + self.install_seccomp(3, 4) + except OSError as error: + os._exit(min(error.errno or errno_module.EIO, 255)) + os._exit(0) + status = _wait_exact_child(pid) + if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0: + code = os.WEXITSTATUS(status) if os.WIFEXITED(status) else errno_module.EIO + raise OSError(code or errno_module.EIO, "seccomp probe failed") + + def _seccomp_program(self, exec_fd: int, setup_error_fd: int) -> list[_SockFilter]: + instructions = [ + _bpf(_BPF_LD_W_ABS, 0, 0, 4), + _bpf(_BPF_JMP_JEQ_K, 1, 0, _AUDIT_ARCH_X86_64), + _bpf(_BPF_RET_K, 0, 0, _SECCOMP_RET_KILL_PROCESS), + _bpf(_BPF_LD_W_ABS, 0, 0, 0), + _bpf(_BPF_JMP_JEQ_K, 0, 9, _SYS_EXECVEAT_X86_64), + _bpf(_BPF_LD_W_ABS, 0, 0, 16), + _bpf(_BPF_JMP_JEQ_K, 0, 0, exec_fd), + _bpf(_BPF_LD_W_ABS, 0, 0, 20), + _bpf(_BPF_JMP_JEQ_K, 0, 0, 0), + _bpf(_BPF_LD_W_ABS, 0, 0, 48), + _bpf(_BPF_JMP_JEQ_K, 0, 0, _AT_EMPTY_PATH), + _bpf(_BPF_LD_W_ABS, 0, 0, 52), + _bpf(_BPF_JMP_JEQ_K, 0, 0, 0), + _bpf(_BPF_RET_K, 0, 0, _SECCOMP_RET_ALLOW), + ] + restricted_start = len(instructions) + # glibc may query or tighten this process's own limits after exec. A + # foreign PID would instead give the evaluator authority over another + # same-UID process, so both words of pid_t must encode the kernel's + # canonical self selector (zero). + instructions.extend( + ( + _bpf(_BPF_JMP_JEQ_K, 0, 5, _SYS_PRLIMIT64_X86_64), + _bpf(_BPF_LD_W_ABS, 0, 0, 16), + _bpf(_BPF_JMP_JEQ_K, 0, 0, 0), + _bpf(_BPF_LD_W_ABS, 0, 0, 20), + _bpf(_BPF_JMP_JEQ_K, 0, 0, 0), + _bpf(_BPF_RET_K, 0, 0, _SECCOMP_RET_ALLOW), + ) + ) + generic_start = len(instructions) + # setup_error_fd is CLOEXEC, so this write capability disappears at the + # successful exec boundary together with the descriptor itself. + generic = tuple(self._runtime_syscalls) + for syscall_number in generic: + instructions.extend( + ( + _bpf(_BPF_JMP_JEQ_K, 0, 1, syscall_number), + _bpf(_BPF_RET_K, 0, 0, _SECCOMP_RET_ALLOW), + ) + ) + final_kill = len(instructions) + instructions.append(_bpf(_BPF_RET_K, 0, 0, _SECCOMP_RET_KILL_PROCESS)) + for index in (6, 8, 10, 12, restricted_start + 2, restricted_start + 4): + distance = final_kill - index - 1 + instructions[index].jf = distance + + # A plain write rule is safe only because close_range leaves fd 1, 2 + # and the CLOEXEC setup fd. No executable or filesystem fd survives. + if setup_error_fd not in (4,): + raise OSError(errno_module.EINVAL, "noncanonical setup fd") + if restricted_start != 14 or generic_start != 20: + raise AssertionError("seccomp branch offset drift") + return instructions + + +def _bpf(code: int, jt: int, jf: int, value: int) -> _SockFilter: + if not 0 <= jt <= 255 or not 0 <= jf <= 255: + raise ValueError("BPF jump exceeds classic filter encoding") + return _SockFilter(code, jt, jf, value) + + +def _read_exact_fd(fd: int, size: int) -> bytes: + chunks = bytearray() + while len(chunks) < size: + try: + chunk = os.read(fd, size - len(chunks)) + except InterruptedError: + continue + if not chunk: + break + chunks.extend(chunk) + return bytes(chunks) + + +def _wait_exact_child(pid: int) -> int: + while True: + try: + waited, status = os.waitpid(pid, 0) + except InterruptedError: + continue + if waited != pid: + raise OSError(errno_module.ECHILD, "wrong child reaped") + return status + + +_CGROUP_NAMES = itertools.count() +_CGROUP_ROOT_V1 = Path("/sys/fs/cgroup") +# One observer plus one child is the whole process tree. The kernel pids +# controller makes thread creation and fork contend for the same final slot. +_OBSERVER_SUBTREE_TASK_LIMIT_V1 = 2 + + +def _current_unified_cgroup_v1() -> Path: + descriptor = os.open( + "/proc/self/cgroup", + os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW, + ) + try: + raw = os.read(descriptor, 4097) + finally: + os.close(descriptor) + if len(raw) > 4096 or not raw.endswith(b"\n") or raw.count(b"\n") != 1: + raise OSError(errno_module.EPROTO, "noncanonical unified cgroup record") + prefix = b"0::" + if not raw.startswith(prefix): + raise OSError(errno_module.ENOTSUP, "unified cgroup v2 is required") + try: + relative = raw[len(prefix) : -1].decode("ascii") + except UnicodeDecodeError as error: + raise OSError(errno_module.EPROTO, "non-ASCII cgroup path") from error + if ( + not relative.startswith("/") + or relative != posixpath.normpath(relative) + or any(part in ("", ".", "..") for part in relative[1:].split("/")) + ): + raise OSError(errno_module.EPROTO, "noncanonical cgroup path") + return _CGROUP_ROOT_V1 / relative[1:] + + +class _CgroupV2V1: + def __init__(self, parent_fd: int, directory_fd: int, name: bytes) -> None: + self._parent_fd = parent_fd + self._directory_fd = directory_fd + self._name = name + + @classmethod + def probe_observer_task_budget(cls, parent: Path) -> None: + parent_fd = os.open( + os.fsencode(parent), + os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW, + ) + current_fd = -1 + current_parent_fd = -1 + try: + current = _current_unified_cgroup_v1() + current_fd = os.open( + os.fsencode(current), + os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW, + ) + current_parent_fd = os.open( + os.fsencode(current.parent), + os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW, + ) + parent_stat = os.fstat(parent_fd) + current_parent_stat = os.fstat(current_parent_fd) + if ( + parent_stat.st_dev != current_parent_stat.st_dev + or parent_stat.st_ino != current_parent_stat.st_ino + ): + raise OSError( + errno_module.EXDEV, + "observer must be in a direct child of the delegated parent", + ) + expected_limit = f"{_OBSERVER_SUBTREE_TASK_LIMIT_V1}\n".encode("ascii") + if ( + _read_cgroup_file(parent_fd, b"pids.max") != expected_limit + or _read_cgroup_file(parent_fd, b"pids.current") != b"1\n" + or _read_cgroup_file(current_fd, b"pids.current") != b"1\n" + ): + raise OSError( + errno_module.EBUSY, + "observer subtree must contain exactly one of two permitted tasks", + ) + finally: + if current_parent_fd >= 0: + os.close(current_parent_fd) + if current_fd >= 0: + os.close(current_fd) + os.close(parent_fd) + + @classmethod + def create( + cls, + parent: Path, + *, + memory_max: int | None, + pids_max: int, + ) -> "_CgroupV2V1": + parent_fd = os.open( + os.fsencode(parent), + os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW, + ) + name = f"labcolors-executor-{os.getpid()}-{next(_CGROUP_NAMES)}".encode("ascii") + directory_fd = -1 + try: + controllers = set(_read_cgroup_file(parent_fd, b"cgroup.controllers").split()) + subtree = set(_read_cgroup_file(parent_fd, b"cgroup.subtree_control").split()) + if not {b"memory", b"pids"} <= controllers or not {b"memory", b"pids"} <= subtree: + raise OSError(errno_module.ENOTSUP, "memory/pids controllers are not delegated") + os.mkdir(name, mode=0o700, dir_fd=parent_fd) + directory_fd = os.open( + name, + os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW, + dir_fd=parent_fd, + ) + group = cls(parent_fd, directory_fd, name) + group._write(b"memory.max", b"max" if memory_max is None else str(memory_max).encode("ascii")) + group._write(b"memory.swap.max", b"0") + group._write(b"memory.oom.group", b"1") + group._write(b"pids.max", str(pids_max).encode("ascii")) + group._require_applied_limits( + memory_max=memory_max, + pids_max=pids_max, + ) + group._require_writable(b"cgroup.kill") + group.oom_kill_count() + group.populated() + return group + except BaseException: + if directory_fd >= 0: + os.close(directory_fd) + try: + os.rmdir(name, dir_fd=parent_fd) + except OSError: + pass + os.close(parent_fd) + raise + + @classmethod + def probe(cls, parent: Path) -> None: + group = cls.create(parent, memory_max=None, pids_max=1) + group.close() + + def attach(self, pid: int) -> None: + self._write(b"cgroup.procs", str(pid).encode("ascii")) + + def kill_all(self) -> None: + self._write(b"cgroup.kill", b"1") + + def oom_kill_count(self) -> int: + values = _parse_cgroup_kv(self._read_required(b"memory.events.local")) + try: + return values[b"oom_kill"] + except KeyError as error: + raise OSError(errno_module.EPROTO, "oom_kill counter missing") from error + + def populated(self) -> bool: + values = _parse_cgroup_kv(self._read_required(b"cgroup.events")) + value = values.get(b"populated") + if value not in (0, 1): + raise OSError(errno_module.EPROTO, "invalid populated counter") + return bool(value) + + def close(self) -> None: + directory_fd, parent_fd = self._directory_fd, self._parent_fd + self._directory_fd = -1 + self._parent_fd = -1 + try: + os.close(directory_fd) + os.rmdir(self._name, dir_fd=parent_fd) + finally: + os.close(parent_fd) + + def _write(self, name: bytes, value: bytes) -> None: + fd = os.open(name, os.O_WRONLY | os.O_CLOEXEC | os.O_NOFOLLOW, dir_fd=self._directory_fd) + try: + written = os.write(fd, value) + if written != len(value): + raise OSError(errno_module.EIO, "short cgroup write") + finally: + os.close(fd) + + def _read_required(self, name: bytes) -> bytes: + return _read_cgroup_file(self._directory_fd, name) + + def _require_writable(self, name: bytes) -> None: + fd = os.open( + name, + os.O_WRONLY | os.O_CLOEXEC | os.O_NOFOLLOW, + dir_fd=self._directory_fd, + ) + os.close(fd) + + def _require_applied_limits( + self, + *, + memory_max: int | None, + pids_max: int, + ) -> None: + expected = { + b"memory.max": b"max" if memory_max is None else str(memory_max).encode("ascii"), + b"memory.swap.max": b"0", + b"memory.oom.group": b"1", + b"pids.max": str(pids_max).encode("ascii"), + } + for name, value in expected.items(): + if self._read_required(name) != value + b"\n": + raise OSError(errno_module.EPROTO, f"cgroup rejected exact {name!r}") + + +def _read_cgroup_file(directory_fd: int, name: bytes) -> bytes: + fd = os.open(name, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW, dir_fd=directory_fd) + try: + chunks = bytearray() + while True: + chunk = os.read(fd, 4096) + if not chunk: + return bytes(chunks) + chunks.extend(chunk) + if len(chunks) > 65536: + raise OSError(errno_module.EOVERFLOW, "cgroup control file too large") + finally: + os.close(fd) + + +def _parse_cgroup_kv(data: bytes) -> dict[bytes, int]: + result: dict[bytes, int] = {} + for line in data.splitlines(): + parts = line.split(b" ") + if len(parts) != 2 or not parts[0] or not parts[1].isdigit() or parts[0] in result: + raise OSError(errno_module.EPROTO, "invalid cgroup counter file") + result[parts[0]] = int(parts[1]) + if not result: + raise OSError(errno_module.EPROTO, "empty cgroup counter file") + return result + + +def _append_bounded_v1(captured: bytearray, chunk: bytes, limit: int) -> bool: + if ( + type(captured) is not bytearray + or type(chunk) is not bytes + or type(limit) is not int + or limit < 0 + or len(captured) > limit + ): + raise ValueError("invalid bounded capture state") + remaining = limit - len(captured) + captured.extend(chunk[:remaining]) + return len(chunk) > remaining + + +def _classify_process_v1( + *, + digest: bytes, + stdout: bytes, + stderr: bytes, + child_status: int | None, + oom_kill_delta: int, + residual: bool, + setup_packet: bytes, + terminal: tuple[str, OutputStreamV1 | None] | None, + limits: ExecutionLimitsV1, +) -> ExecutionResultV1: + if ( + type(digest) is not bytes + or len(digest) != 32 + or type(stdout) is not bytes + or type(stderr) is not bytes + or len(stdout) > limits.max_stdout_bytes + or len(stderr) > limits.max_stderr_bytes + or type(oom_kill_delta) is not int + or oom_kill_delta < 0 + or type(residual) is not bool + or type(setup_packet) is not bytes + ): + return ObserverFailureV1(ObserverReasonV1.BACKEND_CONTRACT) + if terminal is not None: + if terminal == ("timeout", None): + return TimedOutV1(digest, stdout, stderr, limits.wall_timeout_ns) + kind, stream = terminal + if kind != "output" or type(stream) is not OutputStreamV1: + return ObserverFailureV1(ObserverReasonV1.BACKEND_CONTRACT) + limit = ( + limits.max_stdout_bytes + if stream is OutputStreamV1.STDOUT + else limits.max_stderr_bytes + ) + captured = stdout if stream is OutputStreamV1.STDOUT else stderr + if len(captured) != limit: + return ObserverFailureV1(ObserverReasonV1.BACKEND_CONTRACT) + return OutputLimitExceededV1(digest, stdout, stderr, stream, limit) + if setup_packet: + try: + child_error = _parse_child_error_packet_v1(setup_packet) + except ObserverProtocolErrorV1: + return ObserverFailureV1(ObserverReasonV1.CHILD_PROTOCOL) + return SandboxSetupFailedV1( + digest, + stdout, + stderr, + child_error.stage, + child_error.errno, + ) + if residual: + return ResidualProcessesV1(digest, stdout, stderr) + if oom_kill_delta > 0: + return OomKilledV1(digest, stdout, stderr, oom_kill_delta) + if child_status is None: + return ObserverFailureV1(ObserverReasonV1.BACKEND_CONTRACT) + if os.WIFSIGNALED(child_status): + core_dumped = bool(os.WCOREDUMP(child_status)) if hasattr(os, "WCOREDUMP") else False + return SignaledV1( + digest, + stdout, + stderr, + os.WTERMSIG(child_status), + core_dumped, + ) + if not os.WIFEXITED(child_status): + return ObserverFailureV1(ObserverReasonV1.BACKEND_CONTRACT) + exit_code = os.WEXITSTATUS(child_status) + if exit_code: + return ExitNonZeroV1(digest, stdout, stderr, exit_code) + return CompletedV1(digest, stdout, stderr) + + +class NativeLinuxBackendV1: + """Native backend for a dedicated, single-threaded Linux helper process. + + Correctness requires a dedicated helper whose delegated cgroup permits + exactly the observer and one controlled child across the whole subtree. + The kernel pids controller then arbitrates thread creation against fork, + eliminating the observation-to-fork race rather than timing around it. + Native threads created outside CPython and instruction-level inputs such as + CPUID/RDTSC or auxv remain outside this observation boundary, so this result + alone cannot establish ambient-free reproducibility. + """ + + def __init__( + self, + cgroup_parent: str | os.PathLike[str] | None = None, + *, + platform_name: str | None = None, + machine_name: str | None = None, + operations: _NativeLinuxOperationsV1 | None = None, + cgroup_factory: object = _CgroupV2V1, + monotonic_ns: object = time.monotonic_ns, + ) -> None: + self._cgroup_parent = None if cgroup_parent is None else Path(cgroup_parent) + self._platform_name = sys.platform if platform_name is None else platform_name + self._machine_name = platform.machine() if machine_name is None else machine_name + self._operations = operations + self._cgroup_factory = cgroup_factory + self._monotonic_ns = monotonic_ns + + def probe(self, guard: _ProbeGuardV1) -> CapabilityReportV1: + if type(guard) is not _ProbeGuardV1 or not guard.is_current(): + return _invalidated_capability_report_v1() + try: + return self._probe_capability_v1(guard) + except Exception: + return UnsupportedV1( + ( + CapabilityFailureV1( + CapabilityReasonV1.KERNEL_API_UNAVAILABLE, + None, + ), + ) + ) + + def _probe_capability_v1(self, guard: _ProbeGuardV1) -> CapabilityReportV1: + if self._platform_name != "linux": + return UnsupportedV1( + (CapabilityFailureV1(CapabilityReasonV1.HOST_NOT_LINUX, None),) + ) + if self._machine_name.lower() not in ("x86_64", "amd64"): + return UnsupportedV1( + ( + CapabilityFailureV1( + CapabilityReasonV1.ARCHITECTURE_NOT_SUPPORTED, + None, + ), + ) + ) + if self._cgroup_parent is None: + return UnsupportedV1( + ( + CapabilityFailureV1( + CapabilityReasonV1.CGROUP_PARENT_NOT_DECLARED, + None, + ), + ) + ) + if not self._cgroup_parent.is_absolute(): + return UnsupportedV1( + ( + CapabilityFailureV1( + CapabilityReasonV1.CGROUP_V2_UNAVAILABLE, + errno_module.EINVAL, + ), + ) + ) + operations = self._operations + if operations is None: + if sys.platform != "linux": + return UnsupportedV1( + ( + CapabilityFailureV1( + CapabilityReasonV1.KERNEL_API_UNAVAILABLE, + None, + ), + ) + ) + operations = _NativeLinuxOperationsV1() + self._operations = operations + + failures: list[CapabilityFailureV1] = [] + _probe_operation( + operations.probe_standard_fds, + CapabilityReasonV1.STANDARD_FDS_UNAVAILABLE, + failures, + ) + if failures or not guard.is_current(): + if not failures: + return _invalidated_capability_report_v1() + return UnsupportedV1(tuple(failures)) + _probe_operation( + operations.probe_single_threaded, + CapabilityReasonV1.OBSERVER_NOT_SINGLE_THREADED, + failures, + ) + if failures or not guard.is_current(): + if not failures: + return _invalidated_capability_report_v1() + return UnsupportedV1(tuple(failures)) + _probe_operation( + lambda: self._cgroup_factory.probe_observer_task_budget( + self._cgroup_parent + ), + CapabilityReasonV1.OBSERVER_TASK_BUDGET_UNAVAILABLE, + failures, + ) + if failures or not guard.is_current(): + if not failures: + return _invalidated_capability_report_v1() + return UnsupportedV1(tuple(failures)) + self._probe_sealed_memfd(operations, failures) + if failures or not guard.is_current(): + if not failures: + return _invalidated_capability_report_v1() + return UnsupportedV1(tuple(failures)) + _probe_operation( + operations.probe_execveat, + CapabilityReasonV1.EXECVEAT_UNAVAILABLE, + failures, + ) + if failures or not guard.is_current(): + if not failures: + return _invalidated_capability_report_v1() + return UnsupportedV1(tuple(failures)) + _probe_operation( + operations.probe_close_range, + CapabilityReasonV1.CLOSE_RANGE_UNAVAILABLE, + failures, + ) + if failures or not guard.is_current(): + if not failures: + return _invalidated_capability_report_v1() + return UnsupportedV1(tuple(failures)) + _probe_operation( + operations.probe_single_threaded, + CapabilityReasonV1.OBSERVER_NOT_SINGLE_THREADED, + failures, + ) + if failures or not guard.is_current(): + if not failures: + return _invalidated_capability_report_v1() + return UnsupportedV1(tuple(failures)) + _probe_operation( + operations.probe_namespaces, + CapabilityReasonV1.NETWORK_NAMESPACE_UNAVAILABLE, + failures, + ) + if failures or not guard.is_current(): + if not failures: + return _invalidated_capability_report_v1() + return UnsupportedV1(tuple(failures)) + _probe_operation( + operations.probe_single_threaded, + CapabilityReasonV1.OBSERVER_NOT_SINGLE_THREADED, + failures, + ) + if failures or not guard.is_current(): + if not failures: + return _invalidated_capability_report_v1() + return UnsupportedV1(tuple(failures)) + _probe_operation( + operations.probe_seccomp, + CapabilityReasonV1.SECCOMP_FILTER_UNAVAILABLE, + failures, + ) + if failures or not guard.is_current(): + if not failures: + return _invalidated_capability_report_v1() + return UnsupportedV1(tuple(failures)) + _probe_operation( + lambda: self._cgroup_factory.probe(self._cgroup_parent), + CapabilityReasonV1.CGROUP_V2_UNAVAILABLE, + failures, + ) + if failures or not guard.is_current(): + if not failures: + return _invalidated_capability_report_v1() + return UnsupportedV1(tuple(failures)) + return SupportedV1("linux-x86_64", SANDBOX_POLICY_RELEASE_V1) + + def _probe_sealed_memfd( + self, + operations: _NativeLinuxOperationsV1, + failures: list[CapabilityFailureV1], + ) -> None: + try: + fd = operations.create_executable_memfd() + except OSError as error: + failures.append( + CapabilityFailureV1( + CapabilityReasonV1.EXECUTABLE_MEMFD_UNAVAILABLE, + error.errno or None, + ) + ) + return + try: + operations.write_all(fd, b"probe") + operations.make_executable(fd) + operations.add_seals(fd, REQUIRED_FILE_SEALS_V1) + if operations.get_seals(fd) & REQUIRED_FILE_SEALS_V1 != REQUIRED_FILE_SEALS_V1: + raise OSError(errno_module.ENOTSUP, "required file seals missing") + except OSError as error: + failures.append( + CapabilityFailureV1( + CapabilityReasonV1.FILE_SEALS_UNAVAILABLE, + error.errno or None, + ) + ) + finally: + operations.close(fd) + + def run( + self, + request: ExecutionRequestV1, + capability: SupportedV1, + ) -> ExecutionResultV1: + operations = self._operations + if operations is None or self._cgroup_parent is None: + return ObserverFailureV1(ObserverReasonV1.PROBE_FAILED) + + try: + sealed = _seal_executable_v1(request.executable, operations) + except OSError as error: + return SandboxSetupFailedV1( + None, + b"", + b"", + SetupStageV1.SEALED_EXECUTABLE, + error.errno or errno_module.EIO, + ) + try: + return self._run_sealed(request, sealed, operations) + finally: + operations.close(sealed.fd) + + def _run_sealed( + self, + request: ExecutionRequestV1, + sealed: _SealedExecutableV1, + operations: _NativeLinuxOperationsV1, + ) -> ExecutionResultV1: + try: + cwd_fd = os.open( + request.cwd, + os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW, + ) + except OSError as error: + return SandboxSetupFailedV1( + sealed.sha256, + b"", + b"", + SetupStageV1.CWD, + error.errno or errno_module.EIO, + ) + try: + group = self._cgroup_factory.create( + self._cgroup_parent, + memory_max=request.limits.memory_max_bytes, + pids_max=request.limits.pids_max, + ) + except OSError as error: + os.close(cwd_fd) + return SandboxSetupFailedV1( + sealed.sha256, + b"", + b"", + SetupStageV1.CGROUP_CREATE, + error.errno or errno_module.EIO, + ) + try: + try: + result = self._fork_and_observe(request, sealed, operations, cwd_fd, group) + except Exception: + result = ObserverFailureV1(ObserverReasonV1.BACKEND_EXCEPTION) + finally: + os.close(cwd_fd) + cleanup_failed = False + try: + if group.populated(): + group.kill_all() + cleanup_deadline = self._clock() + 1_000_000_000 + while group.populated() and self._clock() < cleanup_deadline: + time.sleep(0.001) + if group.populated(): + cleanup_failed = True + except OSError: + cleanup_failed = True + try: + group.close() + except OSError: + cleanup_failed = True + if cleanup_failed: + return ObserverFailureV1(ObserverReasonV1.CLEANUP_FAILED) + return result + + def _fork_and_observe( + self, + request: ExecutionRequestV1, + sealed: _SealedExecutableV1, + operations: _NativeLinuxOperationsV1, + cwd_fd: int, + group: _CgroupV2V1, + ) -> ExecutionResultV1: + all_fds: list[int] = [] + try: + for _ in range(5): + all_fds.extend(operations.pipe_cloexec()) + except OSError as error: + _close_many(all_fds) + return SandboxSetupFailedV1( + sealed.sha256, + b"", + b"", + SetupStageV1.FILE_DESCRIPTORS, + error.errno or errno_module.EIO, + ) + ( + stdin_read, + stdin_write, + stdout_read, + stdout_write, + stderr_read, + stderr_write, + setup_read, + setup_write, + start_read, + start_write, + ) = all_fds + try: + # The task count is observational; the delegated pids.max=2 + # subtree is the atomic law. Once the observer occupies one slot, + # either a new thread or the controlled child can claim the other, + # never both. + operations.probe_single_threaded() + if self._cgroup_parent is None: + raise OSError(errno_module.EINVAL, "missing cgroup parent") + self._cgroup_factory.probe_observer_task_budget( + self._cgroup_parent + ) + except OSError as error: + _close_many(all_fds) + return SandboxSetupFailedV1( + sealed.sha256, + b"", + b"", + SetupStageV1.OBSERVER_PRECONDITION, + error.errno or errno_module.EIO, + ) + try: + pid = os.fork() + except OSError as error: + _close_many(all_fds) + return SandboxSetupFailedV1( + sealed.sha256, + b"", + b"", + SetupStageV1.CGROUP_ATTACH, + error.errno or errno_module.EIO, + ) + if pid == 0: + self._child( + request, + sealed, + operations, + cwd_fd, + stdin_read, + stdout_write, + stderr_write, + setup_write, + start_read, + ) + os._exit(127) + + _close_many((stdin_read, stdout_write, stderr_write, setup_write, start_read)) + try: + baseline_oom = group.oom_kill_count() + group.attach(pid) + except OSError as error: + _close_many((stdin_write, stdout_read, stderr_read, setup_read, start_write)) + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + _wait_exact_child(pid) + return SandboxSetupFailedV1( + sealed.sha256, + b"", + b"", + SetupStageV1.CGROUP_ATTACH, + error.errno or errno_module.EIO, + ) + try: + os.write(start_write, b"1") + except OSError as error: + try: + group.kill_all() + finally: + _close_many((stdin_write, stdout_read, stderr_read, setup_read, start_write)) + _wait_exact_child(pid) + return SandboxSetupFailedV1( + sealed.sha256, + b"", + b"", + SetupStageV1.CGROUP_ATTACH, + error.errno or errno_module.EIO, + ) + os.close(start_write) + return self._observe( + request, + sealed.sha256, + pid, + group, + baseline_oom, + stdin_write, + stdout_read, + stderr_read, + setup_read, + ) + + def _child( + self, + request: ExecutionRequestV1, + sealed: _SealedExecutableV1, + operations: _NativeLinuxOperationsV1, + cwd_fd: int, + stdin_read: int, + stdout_write: int, + stderr_write: int, + setup_write: int, + start_read: int, + ) -> None: + try: + if _read_exact_fd(start_read, 1) != b"1": + _child_fail(setup_write, SetupStageV1.CGROUP_ATTACH, errno_module.EPIPE) + os.fchdir(cwd_fd) + os.umask(request.umask) + except OSError as error: + _child_fail(setup_write, SetupStageV1.CWD, error.errno or errno_module.EIO) + try: + operations.enter_namespaces() + except OSError as error: + _child_fail(setup_write, SetupStageV1.NAMESPACE, error.errno or errno_module.EIO) + try: + operations.make_mounts_private() + except OSError as error: + _child_fail( + setup_write, + SetupStageV1.MOUNT_PROPAGATION, + error.errno or errno_module.EIO, + ) + try: + protected = tuple( + fcntl.fcntl(fd, fcntl.F_DUPFD_CLOEXEC, 10) + for fd in (sealed.fd, setup_write) + ) + os.dup2(stdin_read, 0) + os.dup2(stdout_write, 1) + os.dup2(stderr_write, 2) + os.dup2(protected[0], 3, inheritable=False) + os.dup2(protected[1], 4, inheritable=False) + operations.close_range_after_setup() + except OSError as error: + _child_fail(setup_write, SetupStageV1.FILE_DESCRIPTORS, error.errno or errno_module.EIO) + try: + _reset_signal_state() + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + operations.set_not_dumpable() + except OSError as error: + _child_fail(4, SetupStageV1.SIGNAL_STATE, error.errno or errno_module.EIO) + except (ValueError, RuntimeError): + _child_fail(4, SetupStageV1.SIGNAL_STATE, errno_module.EINVAL) + try: + operations.set_no_new_privileges() + except OSError as error: + _child_fail(4, SetupStageV1.NO_NEW_PRIVILEGES, error.errno or errno_module.EIO) + try: + operations.install_seccomp(3, 4) + except OSError as error: + _child_fail(4, SetupStageV1.SECCOMP, error.errno or errno_module.EIO) + try: + _SealedExecutableV1(3, sealed.size, sealed.sha256).execveat( + request.argv, + request.environment, + operations, + ) + except OSError as error: + _child_fail(4, SetupStageV1.EXECVEAT, error.errno or errno_module.EIO) + + def _observe( + self, + request: ExecutionRequestV1, + digest: bytes, + pid: int, + group: _CgroupV2V1, + baseline_oom: int, + stdin_fd: int, + stdout_fd: int, + stderr_fd: int, + setup_fd: int, + ) -> ExecutionResultV1: + streams = { + "stdout": bytearray(), + "stderr": bytearray(), + "setup": bytearray(), + } + limits = { + "stdout": request.limits.max_stdout_bytes, + "stderr": request.limits.max_stderr_bytes, + "setup": _CHILD_PACKET.size, + } + fd_by_tag = {"stdin": stdin_fd, "stdout": stdout_fd, "stderr": stderr_fd, "setup": setup_fd} + input_offset = 0 + selector: selectors.BaseSelector | None = None + child_status: int | None = None + terminal: tuple[str, OutputStreamV1 | None] | None = None + observer_failure: ObserverReasonV1 | None = None + killed = False + + try: + selector = selectors.DefaultSelector() + for fd in fd_by_tag.values(): + os.set_blocking(fd, False) + selector.register(stdout_fd, selectors.EVENT_READ, "stdout") + selector.register(stderr_fd, selectors.EVENT_READ, "stderr") + selector.register(setup_fd, selectors.EVENT_READ, "setup") + if request.stdin: + selector.register(stdin_fd, selectors.EVENT_WRITE, "stdin") + else: + os.close(stdin_fd) + fd_by_tag["stdin"] = -1 + deadline_ns = self._clock() + request.limits.wall_timeout_ns + + while child_status is None or any(fd_by_tag[tag] >= 0 for tag in ("stdout", "stderr", "setup")): + if child_status is None: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + child_status = status + if fd_by_tag["stdin"] >= 0: + _selector_close(selector, fd_by_tag, "stdin") + now = self._clock() + if child_status is None and terminal is None and now >= deadline_ns: + terminal = ("timeout", None) + try: + group.kill_all() + killed = True + except OSError: + observer_failure = ObserverReasonV1.CGROUP_OBSERVATION + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + wait_seconds = max(0.0, min((deadline_ns - now) / 1_000_000_000, 0.05)) + events = sorted(selector.select(wait_seconds), key=lambda item: str(item[0].data)) + for key, _mask in events: + tag = key.data + if tag == "stdin": + try: + written = os.write(stdin_fd, request.stdin[input_offset:]) + except BlockingIOError: + continue + except BrokenPipeError: + _selector_close(selector, fd_by_tag, "stdin") + continue + input_offset += written + if input_offset == len(request.stdin): + _selector_close(selector, fd_by_tag, "stdin") + continue + + limit = limits[tag] + remaining = max(0, limit - len(streams[tag])) + try: + chunk = os.read(key.fd, min(65536, remaining + 1)) + except BlockingIOError: + continue + if not chunk: + _selector_close(selector, fd_by_tag, tag) + continue + exceeded = _append_bounded_v1(streams[tag], chunk, limit) + if exceeded: + if tag == "setup": + observer_failure = ObserverReasonV1.CHILD_PROTOCOL + elif terminal is None: + terminal = ( + "output", + OutputStreamV1.STDOUT if tag == "stdout" else OutputStreamV1.STDERR, + ) + if not killed: + try: + group.kill_all() + killed = True + except OSError: + observer_failure = ObserverReasonV1.CGROUP_OBSERVATION + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + if observer_failure is not None and child_status is None and not killed: + try: + group.kill_all() + killed = True + except OSError: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + if child_status is not None and not events: + for tag in ("stdout", "stderr", "setup"): + if fd_by_tag[tag] >= 0: + try: + chunk = os.read(fd_by_tag[tag], 1) + except BlockingIOError: + continue + if not chunk: + _selector_close(selector, fd_by_tag, tag) + else: + exceeded = _append_bounded_v1( + streams[tag], + chunk, + limits[tag], + ) + if exceeded and tag == "setup": + observer_failure = ObserverReasonV1.CHILD_PROTOCOL + elif exceeded and terminal is None: + terminal = ( + "output", + OutputStreamV1.STDOUT if tag == "stdout" else OutputStreamV1.STDERR, + ) + except Exception: + observer_failure = ObserverReasonV1.BACKEND_EXCEPTION + if not killed: + try: + group.kill_all() + killed = True + except OSError: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + finally: + if selector is not None: + selector.close() + _close_many(fd for fd in fd_by_tag.values() if fd >= 0) + if child_status is None: + try: + waited, status = os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + waited = pid + status = 0 + if waited == pid: + child_status = status + if child_status is None: + try: + group.kill_all() + except OSError: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + child_status = _wait_exact_child(pid) + + stdout = bytes(streams["stdout"]) + stderr = bytes(streams["stderr"]) + if observer_failure is not None: + return ObserverFailureV1(observer_failure) + try: + oom_delta = group.oom_kill_count() - baseline_oom + residual = group.populated() + except OSError: + return ObserverFailureV1(ObserverReasonV1.CGROUP_OBSERVATION) + if residual: + try: + group.kill_all() + except OSError: + return ObserverFailureV1(ObserverReasonV1.CGROUP_OBSERVATION) + return _classify_process_v1( + digest=digest, + stdout=stdout, + stderr=stderr, + child_status=child_status, + oom_kill_delta=oom_delta, + residual=residual, + setup_packet=bytes(streams["setup"]), + terminal=terminal, + limits=request.limits, + ) + + def _clock(self) -> int: + value = self._monotonic_ns() + if type(value) is not int or value < 0: + raise OSError(errno_module.EIO, "invalid monotonic clock") + return value + + +def _probe_operation( + operation: object, + reason: CapabilityReasonV1, + failures: list[CapabilityFailureV1], +) -> None: + try: + operation() + except OSError as error: + failures.append(CapabilityFailureV1(reason, error.errno or None)) + except Exception: + failures.append(CapabilityFailureV1(reason, None)) + + +def _child_fail(fd: int, stage: SetupStageV1, error_number: int) -> None: + packet = _encode_child_error_packet_v1(stage, error_number) + try: + offset = 0 + while offset < len(packet): + try: + written = os.write(fd, packet[offset:]) + except InterruptedError: + continue + if written <= 0: + break + offset += written + finally: + os._exit(127) + + +def _reset_signal_state() -> None: + for number in signal.valid_signals(): + if number in (signal.SIGKILL, signal.SIGSTOP): + continue + signal.signal(number, signal.SIG_DFL) + signal.pthread_sigmask(signal.SIG_SETMASK, set()) + + +def _selector_close( + selector: selectors.BaseSelector, + fd_by_tag: dict[str, int], + tag: str, +) -> None: + fd = fd_by_tag[tag] + if fd < 0: + return + try: + selector.unregister(fd) + except KeyError: + pass + os.close(fd) + fd_by_tag[tag] = -1 + + +def _close_many(fds: object) -> None: + for fd in tuple(fds): + try: + os.close(fd) + except OSError: + pass diff --git a/proof/region/v1/arb/keys/gmp.asc b/proof/region/v1/arb/keys/gmp.asc new file mode 100644 index 00000000..e93791c6 --- /dev/null +++ b/proof/region/v1/arb/keys/gmp.asc @@ -0,0 +1,36 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQFNBFDrIWMBCgCyyYoTAD/aL6Yl90eSJ1xuFpODTcwyRZsNSUZKSmKwnqXo9LgS +2B00yVZ2nO2OrSmWPiYikTciitv04bAqFaggSstx6hlni6n3h2PL0jXpf9EI6qOO +oKwi2IVtbBnJAhWpfRcAce6WEqvnav6KjuBM3lr8/5GzDV8tm6+X/G/paTnBqTB9 +pBxrH7smB+iRjDt/6ykWkbYLd6uBKzIkAp4HqAZb/aZMvxI28PeWGjZJQYq2nVPf +LroM6Ub/sNlXpv/bmHJusFQjUL368njhZD1+aVLCUfBCCDzvZc3EYt3wBkbmuCiA +xOb9ramHgiVkNENtzXR+sbQHtKRQv/jllY1qxROM2/rWmL+HohdxL5E0VPple2bg +U/zqX0Hg2byb8FbpzPJO5PnBD+1PME3Uirsly4N7XT80OvhXlYe4t+9X0QARAQAB +tCROaWVscyBNw7ZsbGVyIDxuaXNzZUBseXNhdG9yLmxpdS5zZT6JAZUEEwEIAD8C +GwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAFiEENDwv8PvuXsLtvvOZ81mf+CjG +cpgFAl4h6wsFCRacyygACgkQ81mf+CjGcpjoSwoAmooT2ZjT3zA/km9iJ9pDEZov +gOyVlTSZdohKWp5xtI8C59uZuxuHV9iJigyNWnIBVBr8FjL6Zx5paNQ19SllE1bY +xL4J0jw5j0BP0odT5jORkIsylcKHmR+eSqJiSMvHGsd821UTagYcJu6emat+Kcwn +DHkKPjbEoRmi46n5UzIEG+uHv0sGZUjWZshTCQZVBnJj4sDNJl+kCbYTpUs0f2AE +PjKH6pBk56vIKBP/bNWs2Q2s+VdA7/g5A1N0SkaPt3/+qNslu84qRdIFcqc54stm +R//Qa3C1EBxrrT2P3EzzpkHWxO72jaGlwuN6utX+7YuNe5Cy5ls/BSjugKMiRqBE +AYvFmnbKV2eJS1bqTSR+qTzLn+VS88yvdumAHNNOPsJyMmKPxJD08maMCsqOOys3 +TMl5J+Yz5bSPJQAZ7mu5AU0EUOshYwEKAMqU40j7kGpy7r37vZ+Ytk+LPMRSwhED +ZjTDZETv64nkSz39hOnk+dYA2k9PsZLwkmdzo0kl6HoaQyQYbCrk6nsIOyNb2lBn +S8Bb3ReOfKeINr1bRb6bn5f8s87OH6eKz1lx/Xs/3W2mssIuL5M45vfnG3f3qln5 +L4/C5XR0uIhh1VhXd7os0JXQuOESqnndNHBOstM09BWe3QM9hOH8qfXHp3nM5LQw +rhDJso3VYlTqdghBFfJYqSLGNuz76NyBX+O5yT3pV7RuW+foN+p+kbxjNuapEK58 +ujrzcu2UFRnRz7OesPWei6pfYRv8LKUbxDxlQdeKYIn6DpF8f2Q6a1Uf/bTy7+cO +h9Uv9DR28Bd9Tkxfj1ztdjLsHatOWT7ie415oczRpTZjXj5JDL6xHrPJ27t4Yt2q +PNXQJf96SCuNABEBAAGJAXwEGAEIACYCGwwWIQQ0PC/w++5ewu2+85nzWZ/4KMZy +mAUCXiHtDwUJFpzNLAAKCRDzWZ/4KMZymFJeCgCIHV4v0PhMU92bROWeZRUPsMIJ +kSi53NMq7ztneDCTbfksvxGSt3W8yERVj2bpGEYNumOMkopb/INxauW2otmn7/lq +N99toS9UWr26SLSGGw0OO4I/QJVsmPCDeLsdwDiOpuA4tvYrRuYfRvJ2P7839ktT +MZ54Cj1XJtds4LUqEPVW8eFGX8IcqrP1aiLDzYgufQLLmo+OTxhF9iQVBzRgc3PM +V3yVr/yXod4mQJGWU0vt1N0tff6dvQoQwUQswMo5UDz5BSwbSQsp/J7fKRmayQSW +8g05NxluhXDoiPh6r59XgCRgvv8uc0U3Bvu8PqN2dZxiAwQaNEL4WEqfZqzozjlK +aosC2vbrYplaC2IHPARcmDmxioKPJdFjDKdDOorXLTejndVsPK2NW6sB+bh3akNt +3lIXaMiLvAfNoFNnWg== +=FW3C +-----END PGP PUBLIC KEY BLOCK----- diff --git a/proof/region/v1/arb/keys/mpfr.asc b/proof/region/v1/arb/keys/mpfr.asc new file mode 100644 index 00000000..9681e510 --- /dev/null +++ b/proof/region/v1/arb/keys/mpfr.asc @@ -0,0 +1,21 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEYweR+BYJKwYBBAHaRw8BAQdAo8zZnH90b32CtE+OOvk+OgdGxLDRDgm0PC/H +5lwgkm20JFZpbmNlbnQgTGVmZXZyZSA8dmluY2VudEB2aW5jMTcubmV0PoiTBBMW +CAA7AhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAFiEEpTS+P4PiQdkYKArrWDHR +Gg1NsCoFAmMHlsMCGQEACgkQWDHRGg1NsCrGvQD/dN7dyWX1soay9vDjFAkyDX5O +acyJyRc7aiP555IBb8cBALsg/fSngQDyBeFyTb+jPK+N5gjNTdkGyMCnIlG9LqIE +tC1WaW5jZW50IExlZmV2cmUgPFZpbmNlbnQuTGVmZXZyZUBlbnMtbHlvbi5mcj6I +kAQTFggAOBYhBKU0vj+D4kHZGCgK61gx0RoNTbAqBQJjB5YyAhsDBQsJCAcCBhUK +CQgLAgQWAgMBAh4BAheAAAoJEFgx0RoNTbAqcwoA/RGKEwncAU9UtSVEDSNKGNv9 +Qj4cqBrEvweIWYO97iH0AP4tWPrKZtMiOi9lasyyPJAXqqYMgfsxVfYZr1I0taB+ +C7QqVmluY2VudCBMZWZldnJlIDxWaW5jZW50LkxlZmV2cmVAaW5yaWEuZnI+iJAE +ExYIADgWIQSlNL4/g+JB2RgoCutYMdEaDU2wKgUCYweWEAIbAwULCQgHAgYVCgkI +CwIEFgIDAQIeAQIXgAAKCRBYMdEaDU2wKtfBAP4xWrEvbuLr03iPr5yq46ld298r +WTo/L/XghLLcJHDyIQD8DLgv/4A9e8J+y+2VxU/tM9hEEE/OtFipHahlVlqMeQe4 +OARjB5H4EgorBgEEAZdVAQUBAQdAQA+SDNGmtq+LxAUvL1mWCUhicUWCIX8+d3bc +nN34+GkDAQgHiHgEGBYIACAWIQSlNL4/g+JB2RgoCutYMdEaDU2wKgUCYweR+AIb +DAAKCRBYMdEaDU2wKqGeAP4rKkunb9wTjtUyLiaJ6haNOEFnCVj4H06n3FL8f+Hz +tgD/aEyC0d0L3TEMXnGQhELJAYeoTKlUBvzfZ8dqenK0ZAw= +=3az7 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/proof/region/v1/arb/origin.py b/proof/region/v1/arb/origin.py new file mode 100644 index 00000000..e53a417d --- /dev/null +++ b/proof/region/v1/arb/origin.py @@ -0,0 +1,1187 @@ +#!/usr/bin/env python3 +"""Pure admission primitives for scoped source-integrity observations.""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import os +import stat +import tempfile +from dataclasses import dataclass +from datetime import UTC, date, datetime +from enum import StrEnum +from functools import cmp_to_key +from pathlib import Path +from typing import NoReturn, Protocol + +import provenance + + +OPENPGP_V4_FINGERPRINT_BYTES = 20 +ARMOUR_BEGIN = b"-----BEGIN PGP PUBLIC KEY BLOCK-----" +ARMOUR_END = b"-----END PGP PUBLIC KEY BLOCK-----" +CRC24_INITIAL = 0xB704CE +CRC24_POLYNOMIAL = 0x1864CFB + + +class OriginReasonV1(StrEnum): + INVALID_ARMOUR = "invalid_armour" + ARMOUR_CRC_MISMATCH = "armour_crc_mismatch" + INVALID_FINGERPRINT = "invalid_fingerprint" + INVALID_STATUS = "invalid_status" + SIGNATURE_REJECTED = "signature_rejected" + COORDINATE_MISMATCH = "coordinate_mismatch" + VERIFIER_FAILED = "verifier_failed" + VERIFIER_UNAVAILABLE = "verifier_unavailable" + VERIFIER_OUTPUT_LIMIT = "verifier_output_limit" + VERIFIER_TIMEOUT = "verifier_timeout" + CONTENT_RELATION_MISMATCH = "content_relation_mismatch" + + +@dataclass(frozen=True) +class OriginErrorV1(ValueError): + reason: OriginReasonV1 + detail: str + + def __str__(self) -> str: + return f"{self.reason}: {self.detail}" + + +def _fail(reason: OriginReasonV1, detail: str) -> NoReturn: + raise OriginErrorV1(reason, detail) + + +def _crc24(payload: bytes) -> bytes: + value = CRC24_INITIAL + for byte in payload: + value ^= byte << 16 + for _ in range(8): + value <<= 1 + if value & 0x1000000: + value ^= CRC24_POLYNOMIAL + return (value & 0xFFFFFF).to_bytes(3, "big") + + +def decode_public_key_armour(armour: bytes) -> bytes: + """Decode the one canonical ASCII-armour shape stored by this proof lane.""" + + if type(armour) is not bytes or not armour.endswith(b"\n") or b"\r" in armour: + _fail(OriginReasonV1.INVALID_ARMOUR, "armour must be LF-terminated bytes") + try: + text = armour.decode("ascii") + except UnicodeDecodeError: + _fail(OriginReasonV1.INVALID_ARMOUR, "armour is not ASCII") + lines = text.split("\n") + if ( + len(lines) < 7 + or lines[0] != ARMOUR_BEGIN.decode("ascii") + or lines[1] != "" + or lines[-2] != ARMOUR_END.decode("ascii") + or lines[-1] != "" + ): + _fail(OriginReasonV1.INVALID_ARMOUR, "unexpected armour envelope") + body = lines[2:-3] + checksum = lines[-3] + if ( + not body + or any(len(line) != 64 for line in body[:-1]) + or not 1 <= len(body[-1]) <= 64 + or len(body[-1]) % 4 + or not checksum.startswith("=") + or len(checksum) != 5 + ): + _fail(OriginReasonV1.INVALID_ARMOUR, "noncanonical base64 body") + try: + packets = base64.b64decode("".join(body), validate=True) + expected_crc = base64.b64decode(checksum[1:], validate=True) + except (binascii.Error, ValueError): + _fail(OriginReasonV1.INVALID_ARMOUR, "invalid base64") + if not packets or len(expected_crc) != 3: + _fail(OriginReasonV1.INVALID_ARMOUR, "empty packets or invalid CRC") + if _crc24(packets) != expected_crc: + _fail(OriginReasonV1.ARMOUR_CRC_MISMATCH, "CRC-24 mismatch") + return packets + + +@dataclass(frozen=True) +class AcceptedHistoricalSignatureStatusV1: + signer_fingerprint: bytes + signature_unix_time: int + + def __post_init__(self) -> None: + if ( + type(self.signer_fingerprint) is not bytes + or len(self.signer_fingerprint) != OPENPGP_V4_FINGERPRINT_BYTES + or self.signer_fingerprint == bytes(OPENPGP_V4_FINGERPRINT_BYTES) + ): + raise TypeError("invalid signer fingerprint") + if type(self.signature_unix_time) is not int or self.signature_unix_time <= 0: + raise TypeError("invalid signature time") + + +_ALLOWED_STATUS_TAGS = frozenset( + ( + "NEWSIG", + "KEYEXPIRED", + "KEY_CONSIDERED", + "SIG_ID", + "EXPKEYSIG", + "GOODSIG", + "VALIDSIG", + ) +) +_REJECTED_STATUS_TAGS = frozenset( + ( + "BADSIG", + "ERRSIG", + "REVKEYSIG", + "KEYREVOKED", + "NO_PUBKEY", + "NODATA", + "FAILURE", + "ERROR", + ) +) + + +def _fingerprint(value: bytes) -> bytes: + if ( + type(value) is not bytes + or len(value) != OPENPGP_V4_FINGERPRINT_BYTES + or value == bytes(OPENPGP_V4_FINGERPRINT_BYTES) + ): + _fail(OriginReasonV1.INVALID_FINGERPRINT, "expected fingerprint length") + return value + + +def parse_gpgv_status( + status: bytes, expected_fingerprint: bytes +) -> AcceptedHistoricalSignatureStatusV1: + """Accept one historical machine-status shape; stderr has no authority.""" + + expected = _fingerprint(expected_fingerprint) + if ( + type(status) is not bytes + or not status.endswith(b"\n") + or b"\r" in status + or b"\0" in status + ): + _fail(OriginReasonV1.INVALID_STATUS, "status must be LF-terminated bytes") + lines = status[:-1].split(b"\n") + if not lines: + _fail(OriginReasonV1.INVALID_STATUS, "empty status") + + newsig_count = 0 + valid: list[tuple[bytes, int]] = [] + prefix = b"[GNUPG:] " + for line in lines: + if not line.startswith(prefix): + _fail(OriginReasonV1.INVALID_STATUS, "unframed output") + payload = line[len(prefix) :] + tag_bytes, separator, arguments = payload.partition(b" ") + try: + tag = tag_bytes.decode("ascii") + except UnicodeDecodeError: + _fail(OriginReasonV1.INVALID_STATUS, "non-ASCII tag") + if tag in _REJECTED_STATUS_TAGS: + _fail(OriginReasonV1.SIGNATURE_REJECTED, tag) + if tag not in _ALLOWED_STATUS_TAGS: + _fail(OriginReasonV1.INVALID_STATUS, f"unknown tag {tag}") + if tag == "NEWSIG": + newsig_count += 1 + continue + if not separator: + _fail(OriginReasonV1.INVALID_STATUS, f"missing arguments for {tag}") + if tag != "VALIDSIG": + continue + + fields = arguments.split(b" ") + if len(fields) != 10 or any(not item for item in fields): + _fail(OriginReasonV1.INVALID_STATUS, "invalid VALIDSIG fields") + try: + signer = bytes.fromhex(fields[0].decode("ascii")) + primary = bytes.fromhex(fields[9].decode("ascii")) + signature_time = int(fields[2], 10) + date_text = fields[1].decode("ascii") + parsed_date = date.fromisoformat(date_text) + except (UnicodeDecodeError, ValueError, OverflowError): + _fail(OriginReasonV1.INVALID_STATUS, "invalid VALIDSIG coordinate") + try: + timestamp_date = datetime.fromtimestamp(signature_time, UTC).date() + except (OverflowError, OSError, ValueError): + _fail(OriginReasonV1.INVALID_STATUS, "invalid VALIDSIG time range") + if ( + signer != expected + or primary != expected + or signature_time <= 0 + or parsed_date.isoformat() != date_text + or timestamp_date != parsed_date + ): + _fail(OriginReasonV1.SIGNATURE_REJECTED, "foreign signer or time") + valid.append((signer, signature_time)) + + if newsig_count != 1 or len(valid) != 1: + _fail(OriginReasonV1.SIGNATURE_REJECTED, "expected exactly one signature") + return AcceptedHistoricalSignatureStatusV1(valid[0][0], valid[0][1]) + + +def _digest(value: bytes, field: str) -> bytes: + if type(value) is not bytes or len(value) != 32 or value == bytes(32): + raise TypeError(f"invalid {field}") + return value + + +_GPGV_PROCESS_TOKEN = object() +_SIGNATURE_RELATION_TOKEN = object() + + +@dataclass(frozen=True, init=False) +class GpgvProcessObservationV1: + returncode: int + status: bytes + stderr: bytes + source_tree_identity: bytes + archive_sha256: bytes + signature_sha256: bytes + public_key_packets_sha256: bytes + executable_sha256: bytes + version_sha256: bytes + + def __init__( + self, + returncode: int, + status: bytes, + stderr: bytes, + source_tree_identity: bytes, + archive_sha256: bytes, + signature_sha256: bytes, + public_key_packets_sha256: bytes, + executable_sha256: bytes, + version_sha256: bytes, + *, + _token: object, + ) -> None: + if _token is not _GPGV_PROCESS_TOKEN: + raise TypeError("GpgvProcessObservationV1 is created only by run_gpgv") + if type(returncode) is not int or returncode < 0: + raise TypeError("invalid gpgv returncode") + if type(status) is not bytes or len(status) > 64 * 1024: + raise TypeError("invalid gpgv status") + if type(stderr) is not bytes or len(stderr) > 64 * 1024: + raise TypeError("invalid gpgv stderr") + _digest(source_tree_identity, "source tree identity") + _digest(archive_sha256, "source archive digest") + _digest(signature_sha256, "detached signature digest") + _digest(public_key_packets_sha256, "public key packets digest") + _digest(executable_sha256, "gpgv executable digest") + _digest(version_sha256, "gpgv version digest") + object.__setattr__(self, "returncode", returncode) + object.__setattr__(self, "status", status) + object.__setattr__(self, "stderr", stderr) + object.__setattr__(self, "source_tree_identity", source_tree_identity) + object.__setattr__(self, "archive_sha256", archive_sha256) + object.__setattr__(self, "signature_sha256", signature_sha256) + object.__setattr__( + self, + "public_key_packets_sha256", + public_key_packets_sha256, + ) + object.__setattr__(self, "executable_sha256", executable_sha256) + object.__setattr__(self, "version_sha256", version_sha256) + + +@dataclass(frozen=True, init=False) +class _SignatureRelationObservationV1: + archive_sha256: bytes + source_tree_identity: bytes + signature_sha256: bytes + public_key_packets_sha256: bytes + signer_fingerprint: bytes + signature_unix_time: int + verifier_executable_sha256: bytes + verifier_version_sha256: bytes + + def __init__( + self, + archive_sha256: bytes, + source_tree_identity: bytes, + signature_sha256: bytes, + public_key_packets_sha256: bytes, + signer_fingerprint: bytes, + signature_unix_time: int, + verifier_executable_sha256: bytes, + verifier_version_sha256: bytes, + *, + _token: object, + ) -> None: + if _token is not _SIGNATURE_RELATION_TOKEN: + raise TypeError("signature relation is created only by admission") + for field in ( + "archive_sha256", + "source_tree_identity", + "signature_sha256", + "public_key_packets_sha256", + "verifier_executable_sha256", + "verifier_version_sha256", + ): + _digest(locals()[field], field) + AcceptedHistoricalSignatureStatusV1( + signer_fingerprint, + signature_unix_time, + ) + for field in self.__dataclass_fields__: + object.__setattr__(self, field, locals()[field]) + + +class HistoricalPathRecheckedSignatureDiagnosticV1(_SignatureRelationObservationV1): + """Historical signature diagnostic; no current publisher trust is implied.""" + + +def admit_detached_signature_observation( + *, + expected: provenance.SourceReleaseLockV1, + admitted: provenance.SafeSourceArchiveV1, + signature: bytes, + public_key_armour: bytes, + process: GpgvProcessObservationV1, +) -> HistoricalPathRecheckedSignatureDiagnosticV1: + """Replay one historical signature relation as a path-rechecked diagnostic. + + The result records what the invoked verifier reported for project-pinned + bytes. It does not establish current publisher identity, key status, or an + exact sealed verifier execution. + """ + + if type(expected) is not provenance.SourceReleaseLockV1: + raise TypeError("expected must be SourceReleaseLockV1") + if type(admitted) is not provenance.SafeSourceArchiveV1: + raise TypeError("admitted must be SafeSourceArchiveV1") + if type(expected.integrity) is not provenance.DetachedSignaturePolicyV1: + raise TypeError("source must declare a detached signature policy") + if type(signature) is not bytes: + raise TypeError("signature must be bytes") + if type(process) is not GpgvProcessObservationV1: + raise TypeError("process must be a sealed GpgvProcessObservationV1") + archive = admitted.archive_bytes + actual_archive_sha256 = hashlib.sha256(archive).digest() + actual_signature_sha256 = hashlib.sha256(signature).digest() + key_packets = decode_public_key_armour(public_key_armour) + actual_key_packets_sha256 = hashlib.sha256(key_packets).digest() + if ( + admitted.source_lock_identity != expected.identity + or admitted.archive_sha256 != expected.archive_sha256 + or actual_archive_sha256 != expected.archive_sha256 + or len(signature) != expected.integrity.signature_length + or actual_signature_sha256 != expected.integrity.signature_sha256 + or actual_key_packets_sha256 + != expected.integrity.public_key_packets_sha256 + or process.source_tree_identity != admitted.tree_identity + or process.archive_sha256 != actual_archive_sha256 + or process.signature_sha256 != actual_signature_sha256 + or process.public_key_packets_sha256 != actual_key_packets_sha256 + ): + _fail(OriginReasonV1.COORDINATE_MISMATCH, "source, signature, key, or replay") + if process.returncode != 0: + _fail(OriginReasonV1.VERIFIER_FAILED, f"gpgv exit {process.returncode}") + signature_observation = parse_gpgv_status( + process.status, + expected.integrity.signer_fingerprint, + ) + return HistoricalPathRecheckedSignatureDiagnosticV1( + actual_archive_sha256, + admitted.tree_identity, + actual_signature_sha256, + actual_key_packets_sha256, + signature_observation.signer_fingerprint, + signature_observation.signature_unix_time, + process.executable_sha256, + process.version_sha256, + _token=_SIGNATURE_RELATION_TOKEN, + ) + + +def _read_regular_file_descriptor(descriptor: int) -> bytes: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size <= 0: + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "gpgv is not a regular file") + chunks: list[bytes] = [] + offset = 0 + while offset < metadata.st_size: + chunk = os.pread(descriptor, min(64 * 1024, metadata.st_size - offset), offset) + if not chunk: + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "short gpgv read") + chunks.append(chunk) + offset += len(chunk) + return b"".join(chunks) + +@dataclass(frozen=True) +class DiagnosticProcessRequestV1: + """Client-owned diagnostic execution request with no authority semantics.""" + + argv: tuple[str, ...] + stdin: bytes | None + cwd: Path + environment: dict[str, str] + pass_fds: tuple[int, ...] + timeout_seconds: int | float + stdout_limit: int + stderr_limit: int + + def __post_init__(self) -> None: + if ( + type(self.argv) is not tuple + or not self.argv + or any(type(item) is not str or not item for item in self.argv) + or (self.stdin is not None and type(self.stdin) is not bytes) + or not isinstance(self.cwd, Path) + or type(self.environment) is not dict + or any( + type(key) is not str or type(value) is not str + for key, value in self.environment.items() + ) + or type(self.pass_fds) is not tuple + or any(type(fd) is not int or fd < 0 for fd in self.pass_fds) + or type(self.timeout_seconds) not in (int, float) + or self.timeout_seconds <= 0 + or type(self.stdout_limit) is not int + or self.stdout_limit < 0 + or type(self.stderr_limit) is not int + or self.stderr_limit < 0 + ): + raise TypeError("invalid diagnostic process request") + + +@dataclass(frozen=True) +class DiagnosticProcessObservationV1: + """Untrusted bytes returned by client-owned diagnostic execution.""" + + returncode: int + stdout: bytes + stderr: bytes + + def __post_init__(self) -> None: + if ( + type(self.returncode) is not int + or not -(1 << 31) <= self.returncode < 1 << 31 + or type(self.stdout) is not bytes + or type(self.stderr) is not bytes + ): + raise TypeError("invalid diagnostic process observation") + + +class DiagnosticProcessRunnerV1(Protocol): + """Client-owned resource runner; this interface grants no sandbox claim.""" + + def run( + self, + request: DiagnosticProcessRequestV1, + ) -> DiagnosticProcessObservationV1: ... + + +def _observe_diagnostic_process_v1( + runner: DiagnosticProcessRunnerV1, + request: DiagnosticProcessRequestV1, +) -> DiagnosticProcessObservationV1: + try: + observed = runner.run(request) + except Exception: + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "diagnostic runner failed") + if type(observed) is not DiagnosticProcessObservationV1: + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "foreign diagnostic observation") + if ( + len(observed.stdout) > request.stdout_limit + or len(observed.stderr) > request.stderr_limit + ): + _fail(OriginReasonV1.VERIFIER_OUTPUT_LIMIT, "diagnostic output exceeded policy") + return observed + + +def _run_diagnostic_v1( + runner: DiagnosticProcessRunnerV1, + argv: tuple[str, ...], + *, + stdin: bytes | None, + cwd: Path, + environment: dict[str, str], + pass_fds: tuple[int, ...], + timeout_seconds: int | float, + stdout_limit: int, + stderr_limit: int, +) -> DiagnosticProcessObservationV1: + return _observe_diagnostic_process_v1( + runner, + DiagnosticProcessRequestV1( + argv, + stdin, + cwd, + environment, + pass_fds, + timeout_seconds, + stdout_limit, + stderr_limit, + ), + ) + + +def run_gpgv( + source: provenance.SafeSourceArchiveV1, + signature: bytes, + public_key_armour: bytes, + *, + executable: Path, + runner: DiagnosticProcessRunnerV1, +) -> GpgvProcessObservationV1: + """Request a client-owned gpgv diagnostic; never mint execution authority.""" + + if type(source) is not provenance.SafeSourceArchiveV1: + raise TypeError("source must be SafeSourceArchiveV1") + if any(type(value) is not bytes for value in (signature, public_key_armour)): + raise TypeError("gpgv signature and key must be bytes") + archive = source.archive_bytes + key_packets = decode_public_key_armour(public_key_armour) + signature_sha256 = hashlib.sha256(signature).digest() + public_key_packets_sha256 = hashlib.sha256(key_packets).digest() + try: + resolved = executable.resolve(strict=True) + descriptor = os.open( + resolved, + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + except (OSError, RuntimeError): + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "cannot open gpgv") + try: + executable_bytes = _read_regular_file_descriptor(descriptor) + executable_sha256 = hashlib.sha256(executable_bytes).digest() + descriptor_exec_supported = Path("/proc/self/fd").is_dir() + descriptor_path = ( + f"/proc/self/fd/{descriptor}" + if descriptor_exec_supported + else str(resolved) + ) + inherited_descriptors = (descriptor,) if descriptor_exec_supported else () + with tempfile.TemporaryDirectory(prefix="labcolors-gpgv-") as temporary: + root = Path(temporary) + keyring = root / "keyring.gpg" + detached = root / "signature.bin" + keyring.write_bytes(key_packets) + detached.write_bytes(signature) + os.chmod(keyring, 0o400) + os.chmod(detached, 0o400) + environment = { + "HOME": "/nonexistent", + "LANG": "C", + "LC_ALL": "C", + "TZ": "UTC", + } + version = _run_diagnostic_v1( + runner, + (descriptor_path, "--version"), + stdin=None, + cwd=root, + environment=environment, + pass_fds=inherited_descriptors, + timeout_seconds=10, + stdout_limit=64 * 1024, + stderr_limit=64 * 1024, + ) + verified = _run_diagnostic_v1( + runner, + ( + descriptor_path, + "--homedir", + str(root), + "--keyring", + str(keyring), + "--status-fd", + "1", + str(detached), + "-", + ), + stdin=archive, + cwd=root, + environment=environment, + pass_fds=inherited_descriptors, + timeout_seconds=60, + stdout_limit=64 * 1024, + stderr_limit=64 * 1024, + ) + if version.returncode != 0 or not version.stdout or version.stderr: + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "gpgv version failed") + if verified.returncode < 0: + _fail( + OriginReasonV1.VERIFIER_FAILED, + f"gpgv terminated by signal {-verified.returncode}", + ) + if not descriptor_exec_supported: + try: + if hashlib.sha256(resolved.read_bytes()).digest() != executable_sha256: + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "gpgv changed during replay") + except OSError: + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "cannot re-read gpgv") + return GpgvProcessObservationV1( + verified.returncode, + verified.stdout, + verified.stderr, + source.tree_identity, + source.archive_sha256, + signature_sha256, + public_key_packets_sha256, + executable_sha256, + hashlib.sha256(version.stdout).digest(), + _token=_GPGV_PROCESS_TOKEN, + ) + finally: + os.close(descriptor) + + +def _sha1(value: bytes, field: str) -> bytes: + if type(value) is not bytes or len(value) != 20 or value == bytes(20): + raise TypeError(f"invalid {field}") + return value + + +def _source_path(value: str) -> bytes: + if type(value) is not str or not value or value.startswith("/") or "\\" in value: + raise TypeError("invalid source path") + try: + encoded = value.encode("ascii") + except UnicodeEncodeError: + raise TypeError("source path must be ASCII") from None + if ( + len(encoded) > 4096 + or any(byte < 0x20 or byte == 0x7F for byte in encoded) + or any(part in ("", ".", "..") for part in value.split("/")) + ): + raise TypeError("invalid source path") + return encoded + + +@dataclass(frozen=True) +class FileCoordinateV1: + path: str + mode: int + length: int + sha256: bytes + + def __post_init__(self) -> None: + _source_path(self.path) + if type(self.mode) is not int or self.mode not in (0o644, 0o700, 0o755): + raise TypeError("invalid source mode") + if type(self.length) is not int or self.length < 0 or self.length >= 1 << 64: + raise TypeError("invalid source length") + _digest(self.sha256, "source file digest") + + +def _canonical_files(value: tuple[FileCoordinateV1, ...], field: str) -> None: + if type(value) is not tuple or any(type(item) is not FileCoordinateV1 for item in value): + raise TypeError(f"invalid {field}") + paths = tuple(item.path for item in value) + if paths != tuple(sorted(set(paths))): + raise TypeError(f"noncanonical {field}") + + +def _file_set_digest(files: tuple[FileCoordinateV1, ...], label: bytes) -> bytes: + hasher = hashlib.sha256(label) + hasher.update(len(files).to_bytes(8, "big")) + for item in files: + path = item.path.encode("ascii") + hasher.update(len(path).to_bytes(4, "big")) + hasher.update(path) + hasher.update(item.mode.to_bytes(4, "big")) + hasher.update(item.length.to_bytes(8, "big")) + hasher.update(item.sha256) + return hasher.digest() + + +_GIT_PROCESS_TOKEN = object() +_GIT_RELATION_TOKEN = object() + + +@dataclass(frozen=True, init=False) +class GitTreeProcessObservationV1: + commit: bytes + tree: bytes + commit_object_sha256: bytes + files: tuple[FileCoordinateV1, ...] + executable_sha256: bytes + version_sha256: bytes + + def __init__( + self, + commit: bytes, + tree: bytes, + commit_object_sha256: bytes, + files: tuple[FileCoordinateV1, ...], + executable_sha256: bytes, + version_sha256: bytes, + *, + _token: object, + ) -> None: + if _token is not _GIT_PROCESS_TOKEN: + raise TypeError("GitTreeProcessObservationV1 is created only by run_git_tree") + _sha1(commit, "Git commit") + _sha1(tree, "Git tree") + _digest(commit_object_sha256, "Git commit object digest") + _canonical_files(files, "Git files") + if not files or any(item.mode == 0o700 for item in files): + raise TypeError("invalid Git tree") + _digest(executable_sha256, "Git executable digest") + _digest(version_sha256, "Git version digest") + object.__setattr__(self, "commit", commit) + object.__setattr__(self, "tree", tree) + object.__setattr__(self, "commit_object_sha256", commit_object_sha256) + object.__setattr__(self, "files", files) + object.__setattr__(self, "executable_sha256", executable_sha256) + object.__setattr__(self, "version_sha256", version_sha256) + + +@dataclass(frozen=True, init=False) +class RecomputedGitContentRelationV1: + archive_sha256: bytes + source_tree_identity: bytes + commit: bytes + tree: bytes + commit_object_sha256: bytes + git_files_identity: bytes + archive_files_identity: bytes + common_file_count: int + omitted_file_count: int + project_pinned_release_only_file_count: int + + def __init__( + self, + archive_sha256: bytes, + source_tree_identity: bytes, + commit: bytes, + tree: bytes, + commit_object_sha256: bytes, + git_files_identity: bytes, + archive_files_identity: bytes, + common_file_count: int, + omitted_file_count: int, + project_pinned_release_only_file_count: int, + *, + _token: object, + ) -> None: + if _token is not _GIT_RELATION_TOKEN: + raise TypeError("Git relation is created only by admission") + _sha1(commit, "Git commit") + _sha1(tree, "Git tree") + for field in ( + "archive_sha256", + "source_tree_identity", + "git_files_identity", + "archive_files_identity", + "commit_object_sha256", + ): + _digest(locals()[field], field) + for field in ( + "common_file_count", + "omitted_file_count", + "project_pinned_release_only_file_count", + ): + value = locals()[field] + if type(value) is not int or value <= 0: + raise TypeError(f"invalid {field}") + for field in self.__dataclass_fields__: + object.__setattr__(self, field, locals()[field]) + + +def admit_git_content_relation_observation( + *, + expected: provenance.SourceReleaseLockV1, + admitted: provenance.SafeSourceArchiveV1, + process: GitTreeProcessObservationV1, +) -> RecomputedGitContentRelationV1: + """Relate archive bytes to a project-pinned, independently replayed graph. + + Git supplies bytes and diagnostics only. The admitted relation derives + from locally recomputed commit, tree, and blob identities, so executable + metadata is intentionally absent from its identity and authority surface. + """ + + if type(expected) is not provenance.SourceReleaseLockV1: + raise TypeError("expected must be SourceReleaseLockV1") + if type(admitted) is not provenance.SafeSourceArchiveV1: + raise TypeError("admitted must be SafeSourceArchiveV1") + if type(expected.integrity) is not provenance.GitContentRelationPolicyV1: + raise TypeError("source must declare a Git content relation policy") + if type(process) is not GitTreeProcessObservationV1: + raise TypeError("process must be a sealed GitTreeProcessObservationV1") + if ( + admitted.source_lock_identity != expected.identity + or admitted.archive_sha256 != expected.archive_sha256 + or process.commit != expected.integrity.commit + or process.tree != expected.integrity.tree + ): + _fail(OriginReasonV1.CONTENT_RELATION_MISMATCH, "source, commit, or tree") + expected_common_file_count = expected.integrity.common_file_count + omitted_paths = expected.integrity.omitted_paths + project_pinned_release_only_files = tuple( + FileCoordinateV1(item.path, item.mode, item.length, item.sha256) + for item in expected.integrity.project_pinned_release_only_files + ) + archive_files = tuple( + FileCoordinateV1(item.path, item.mode, item.length, item.sha256) + for item in admitted.files + ) + _canonical_files( + project_pinned_release_only_files, + "project-pinned release-only files", + ) + _canonical_files(archive_files, "archive files") + if not project_pinned_release_only_files or not archive_files: + raise TypeError("empty content relation") + if type(omitted_paths) is not tuple or not omitted_paths: + raise TypeError("empty omitted paths") + for path in omitted_paths: + _source_path(path) + if omitted_paths != tuple(sorted(set(omitted_paths))): + raise TypeError("noncanonical omitted paths") + + git_by_path = {item.path: item for item in process.files} + archive_by_path = {item.path: item for item in archive_files} + release_only_by_path = { + item.path: item for item in project_pinned_release_only_files + } + omitted = set(omitted_paths) + release_only = set(release_only_by_path) + git_paths = set(git_by_path) + archive_paths = set(archive_by_path) + common_paths = git_paths - omitted + if ( + len(common_paths) != expected_common_file_count + or not omitted <= git_paths + or omitted & archive_paths + or release_only & git_paths + or not release_only <= archive_paths + or archive_paths != common_paths | release_only + ): + _fail(OriginReasonV1.CONTENT_RELATION_MISMATCH, "path partition") + if any(archive_by_path[path] != git_by_path[path] for path in common_paths): + _fail(OriginReasonV1.CONTENT_RELATION_MISMATCH, "common file content") + if any( + archive_by_path[path] != release_only_by_path[path] + for path in release_only + ): + _fail( + OriginReasonV1.CONTENT_RELATION_MISMATCH, + "project-pinned release-only file content", + ) + return RecomputedGitContentRelationV1( + admitted.archive_sha256, + admitted.tree_identity, + process.commit, + process.tree, + process.commit_object_sha256, + _file_set_digest(process.files, b"labcolors.git-tree-files.v1\0"), + _file_set_digest(archive_files, b"labcolors.release-archive-files.v1\0"), + len(common_paths), + len(omitted), + len(release_only), + _token=_GIT_RELATION_TOKEN, + ) + + +def _git_object_id(value: bytes) -> bytes: + if len(value) != 40: + _fail(OriginReasonV1.INVALID_STATUS, "invalid Git object id") + try: + decoded = bytes.fromhex(value.decode("ascii")) + except (UnicodeDecodeError, ValueError): + _fail(OriginReasonV1.INVALID_STATUS, "invalid Git object id") + if decoded == bytes(20): + _fail(OriginReasonV1.INVALID_STATUS, "zero Git object id") + return decoded + + +def _parse_git_listing(raw: bytes) -> tuple[tuple[bytes, str, int], ...]: + if type(raw) is not bytes or not raw or not raw.endswith(b"\0"): + _fail(OriginReasonV1.INVALID_STATUS, "invalid Git listing") + records: list[tuple[bytes, str, int]] = [] + previous: str | None = None + for encoded in raw[:-1].split(b"\0"): + metadata, separator, path_raw = encoded.partition(b"\t") + fields = metadata.split(b" ") + if not separator or len(fields) != 3 or fields[1] != b"blob": + _fail(OriginReasonV1.INVALID_STATUS, "non-blob Git entry") + if fields[0] == b"100644": + mode = 0o644 + elif fields[0] == b"100755": + mode = 0o755 + else: + _fail(OriginReasonV1.INVALID_STATUS, "unsupported Git mode") + try: + path = path_raw.decode("ascii") + except UnicodeDecodeError: + _fail(OriginReasonV1.INVALID_STATUS, "non-ASCII Git path") + try: + _source_path(path) + except TypeError: + _fail(OriginReasonV1.INVALID_STATUS, "invalid Git path") + _git_object_id(fields[2]) + if previous is not None and previous >= path: + _fail(OriginReasonV1.INVALID_STATUS, "noncanonical Git path order") + previous = path + records.append((fields[2], path, mode)) + if not records: + _fail(OriginReasonV1.INVALID_STATUS, "empty Git tree") + return tuple(records) + + +def _recompute_git_tree_identity( + listing: tuple[tuple[bytes, str, int], ...] +) -> bytes: + """Rebuild recursive Git tree objects without trusting `git ls-tree` IDs.""" + + if type(listing) is not tuple or not listing: + _fail(OriginReasonV1.INVALID_STATUS, "empty Git listing") + root: dict[bytes, object] = {} + for object_id_raw, path, mode in listing: + object_id = _git_object_id(object_id_raw) + components = path.encode("ascii").split(b"/") + node = root + for component in components[:-1]: + existing = node.get(component) + if existing is None: + child: dict[bytes, object] = {} + node[component] = child + node = child + elif type(existing) is dict: + node = existing + else: + _fail(OriginReasonV1.INVALID_STATUS, "Git file/directory collision") + leaf = components[-1] + if leaf in node: + _fail(OriginReasonV1.INVALID_STATUS, "duplicate Git tree entry") + node[leaf] = (mode, object_id) + + def compare_entries( + left: tuple[bytes, bool, bytes, bytes], + right: tuple[bytes, bool, bytes, bytes], + ) -> int: + left_name, left_tree, _left_mode, _left_id = left + right_name, right_tree, _right_mode, _right_id = right + common = min(len(left_name), len(right_name)) + if left_name[:common] != right_name[:common]: + return -1 if left_name[:common] < right_name[:common] else 1 + left_next = left_name[common] if common < len(left_name) else (47 if left_tree else 0) + right_next = right_name[common] if common < len(right_name) else (47 if right_tree else 0) + return left_next - right_next + + # Git permits paths deeper than Python's recursion limit. Explicit + # post-order traversal keeps the accepted path grammar independent of the + # host interpreter stack while preserving Git's byte ordering exactly. + digests: dict[int, bytes] = {} + stack: list[tuple[dict[bytes, object], bool]] = [(root, False)] + while stack: + node, visited = stack.pop() + if not visited: + stack.append((node, True)) + for child in node.values(): + if type(child) is dict: + stack.append((child, False)) + continue + + entries: list[tuple[bytes, bool, bytes, bytes]] = [] + for name, child in node.items(): + if type(child) is dict: + entries.append((name, True, b"40000", digests[id(child)])) + else: + mode, object_id = child # type: ignore[misc] + encoded_mode = b"100644" if mode == 0o644 else b"100755" + entries.append((name, False, encoded_mode, object_id)) + entries.sort(key=cmp_to_key(compare_entries)) + body = b"".join( + mode + b" " + name + b"\0" + object_id + for name, _is_tree, mode, object_id in entries + ) + digests[id(node)] = hashlib.sha1( + b"tree " + str(len(body)).encode("ascii") + b"\0" + body + ).digest() + + return digests[id(root)] + + +def _admit_git_commit_object(body: bytes, commit: bytes, tree: bytes) -> bytes: + if type(body) is not bytes or not body: + _fail(OriginReasonV1.INVALID_STATUS, "empty Git commit object") + expected_commit = _sha1(commit, "Git commit") + expected_tree = _sha1(tree, "Git tree") + header = b"commit " + str(len(body)).encode("ascii") + b"\0" + if hashlib.sha1(header + body).digest() != expected_commit: + _fail(OriginReasonV1.CONTENT_RELATION_MISMATCH, "Git commit identity") + first_line, separator, _remaining = body.partition(b"\n") + if not separator or first_line != b"tree " + expected_tree.hex().encode("ascii"): + _fail(OriginReasonV1.CONTENT_RELATION_MISMATCH, "commit to tree edge") + return hashlib.sha256(body).digest() + + +def _parse_git_batch( + raw: bytes, listing: tuple[tuple[bytes, str, int], ...] +) -> tuple[FileCoordinateV1, ...]: + if type(raw) is not bytes: + raise TypeError("Git batch output must be bytes") + offset = 0 + files: list[FileCoordinateV1] = [] + for object_id_raw, path, mode in listing: + header_end = raw.find(b"\n", offset) + if header_end < 0: + _fail(OriginReasonV1.INVALID_STATUS, "truncated Git batch header") + header = raw[offset:header_end].split(b" ") + if len(header) != 3 or header[0] != object_id_raw or header[1] != b"blob": + _fail(OriginReasonV1.INVALID_STATUS, "foreign Git batch object") + try: + length = int(header[2], 10) + except ValueError: + _fail(OriginReasonV1.INVALID_STATUS, "invalid Git blob length") + if ( + length < 0 + or length >= 1 << 64 + or not header[2].isdigit() + or header[2] != str(length).encode("ascii") + ): + _fail(OriginReasonV1.INVALID_STATUS, "invalid Git blob length") + body_start = header_end + 1 + body_end = body_start + length + if body_end >= len(raw) or raw[body_end : body_end + 1] != b"\n": + _fail(OriginReasonV1.INVALID_STATUS, "truncated Git blob") + body = raw[body_start:body_end] + object_id = _git_object_id(object_id_raw) + object_header = b"blob " + str(length).encode("ascii") + b"\0" + if hashlib.sha1(object_header + body).digest() != object_id: + _fail(OriginReasonV1.CONTENT_RELATION_MISMATCH, "Git blob identity") + files.append(FileCoordinateV1(path, mode, length, hashlib.sha256(body).digest())) + offset = body_end + 1 + if offset != len(raw): + _fail(OriginReasonV1.INVALID_STATUS, "trailing Git batch bytes") + return tuple(files) + + +def run_git_tree( + repository: Path, + expected_commit: bytes, + expected_tree: bytes, + *, + executable: Path, + runner: DiagnosticProcessRunnerV1, +) -> GitTreeProcessObservationV1: + """Parse a client-owned Git diagnostic and recompute every content edge.""" + + commit = _sha1(expected_commit, "expected Git commit") + tree = _sha1(expected_tree, "expected Git tree") + try: + root = repository.resolve(strict=True) + except (OSError, RuntimeError): + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "Git repository unavailable") + if not root.is_dir(): + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "Git repository is not a directory") + try: + resolved = executable.resolve(strict=True) + descriptor = os.open( + resolved, + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + except (OSError, RuntimeError): + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "cannot open Git") + try: + executable_bytes = _read_regular_file_descriptor(descriptor) + executable_sha256 = hashlib.sha256(executable_bytes).digest() + descriptor_exec_supported = Path("/proc/self/fd").is_dir() + executable_path = ( + f"/proc/self/fd/{descriptor}" + if descriptor_exec_supported + else str(resolved) + ) + inherited_descriptors = (descriptor,) if descriptor_exec_supported else () + environment = { + "GIT_CONFIG_GLOBAL": "/dev/null", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_NO_LAZY_FETCH": "1", + "GIT_OPTIONAL_LOCKS": "0", + "GIT_PAGER": "cat", + "HOME": "/nonexistent", + "LANG": "C", + "LC_ALL": "C", + "PATH": "/usr/bin:/bin", + "TZ": "UTC", + } + + def invoke( + arguments: tuple[str, ...], + *, + stdin: bytes | None = None, + timeout: int = 60, + stdout_limit: int = 64 * 1024, + ) -> bytes: + process = _run_diagnostic_v1( + runner, + (executable_path, "-C", str(root), *arguments), + stdin=stdin, + cwd=root, + environment=environment, + pass_fds=inherited_descriptors, + timeout_seconds=timeout, + stdout_limit=stdout_limit, + stderr_limit=64 * 1024, + ) + if process.returncode != 0 or process.stderr: + _fail(OriginReasonV1.VERIFIER_FAILED, "Git command rejected") + return process.stdout + + version_process = _run_diagnostic_v1( + runner, + (executable_path, "--version"), + stdin=None, + cwd=root, + environment=environment, + pass_fds=inherited_descriptors, + timeout_seconds=10, + stdout_limit=64 * 1024, + stderr_limit=64 * 1024, + ) + if version_process.returncode != 0 or not version_process.stdout or version_process.stderr: + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "Git version failed") + + commit_object = invoke( + ("cat-file", "commit", commit.hex()), + stdout_limit=1024 * 1024, + ) + commit_object_sha256 = _admit_git_commit_object(commit_object, commit, tree) + listing = _parse_git_listing( + invoke( + ("ls-tree", "-r", "-z", "--full-tree", tree.hex()), + stdout_limit=64 * 1024 * 1024, + ) + ) + if _recompute_git_tree_identity(listing) != tree: + _fail(OriginReasonV1.CONTENT_RELATION_MISMATCH, "Git tree identity") + query = b"".join(object_id + b"\n" for object_id, _path, _mode in listing) + if len(query) > 1024 * 1024: + _fail(OriginReasonV1.INVALID_STATUS, "oversized Git query") + batch = invoke( + ("cat-file", "--batch"), + stdin=query, + timeout=180, + stdout_limit=128 * 1024 * 1024, + ) + files = _parse_git_batch(batch, listing) + if not descriptor_exec_supported: + try: + if hashlib.sha256(resolved.read_bytes()).digest() != executable_sha256: + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "Git changed during replay") + except OSError: + _fail(OriginReasonV1.VERIFIER_UNAVAILABLE, "cannot re-read Git") + return GitTreeProcessObservationV1( + commit, + tree, + commit_object_sha256, + files, + executable_sha256, + hashlib.sha256(version_process.stdout).digest(), + _token=_GIT_PROCESS_TOKEN, + ) + finally: + os.close(descriptor) diff --git a/proof/region/v1/arb/pipeline.py b/proof/region/v1/arb/pipeline.py new file mode 100644 index 00000000..cfdb6231 --- /dev/null +++ b/proof/region/v1/arb/pipeline.py @@ -0,0 +1,2545 @@ +#!/usr/bin/env python3 +"""Controlled offline BUILD/RUN observations for the Arb evaluator. + +The unsealed Linux x64 host and its Docker daemon are explicitly inside this +V1 trust boundary. Provider identity and host freshness are not observable +here. This module emits neither SLSA nor source-bound receipts: it observes two +fresh-container builds, owns the exact post-exit output bytes, and can feed that +same bytes object to an explicitly diagnostic, unsealed RUN observation. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import platform +import selectors +import signal +import stat +import subprocess +import tempfile +import time +from dataclasses import dataclass, fields +from enum import StrEnum +from functools import cached_property +from pathlib import Path, PurePosixPath +from typing import NoReturn, Protocol, TypeAlias + +import executor +import provenance +import region_proof_protocol as protocol +import snapshot + + +OCI_IMAGE_MANIFEST_SHA256_V1 = ( + "c74b2d34b775e6a1b14b13b1d41dc7233f62a18f7a6a4e139e0cf59eeab2e070" +) +OCI_IMAGE_REFERENCE_V1 = f"gcc@sha256:{OCI_IMAGE_MANIFEST_SHA256_V1}" +OCI_PLATFORM_V1 = "linux/amd64" +EVALUATOR_OUTPUT_NAME_V1 = "arb-evaluator-v1" +GENERATED_FORMULA_PATH_V1 = "generated/formula.generated.c" +FORMULA_SPEC_PATH_V1 = "crates/labcolors-core/contracts/contextual-region-formula-v1.lcir" +FORMULA_GENERATOR_PATH_V1 = "proof/region/v1/arb/evaluator/formula.py" +BUILD_RECIPE_PATH_V1 = "proof/region/v1/arb/build.sh" + +FORMULA_SPEC_SHA256_V1 = "a6f77ac462f226453b1c27bbd8637b62780b9a640c317a6f50028dacd1de8540" +GENERATED_FORMULA_SHA256_V1 = "9958f20c8ca598625db0593a45f8f8bc79e4b2f22b53263b6c32d78a5e1d2693" + +# This is a drift gate, not documentation copied from memory. Admission below +# hashes every exact input and rejects a local source edit until this manifest +# is deliberately updated together with its binding tests. +_PINNED_BUILD_SOURCE_SHA256_V1 = { + FORMULA_SPEC_PATH_V1: FORMULA_SPEC_SHA256_V1, + GENERATED_FORMULA_PATH_V1: GENERATED_FORMULA_SHA256_V1, + BUILD_RECIPE_PATH_V1: "9fadd62db18ecd1a879363c70ad43c08479f34edc536d40eb197177ae78edfe7", + FORMULA_GENERATOR_PATH_V1: "16629cc3a2ef745ae244ae4762f8946a6546972886f96beeb9ee4920b043040c", + "proof/region/v1/arb/evaluator/formula.h": "46fd5ad1b68b728efcd990a71d1dcc273b75e3391d8c06ef2fd0ac6a4d7dfdbd", + "proof/region/v1/arb/evaluator/hash.c": "c28e6281208f09ca15fa74aea0091f27726ed68efc3480c34a7db33b8ca3567e", + "proof/region/v1/arb/evaluator/hash.h": "a62c07f2eca9294b4c1c802e2a9e6cff6ad9f8fd696a74b54a21489d56fab6c4", + "proof/region/v1/arb/evaluator/interval.c": "93f206258b83fc0f373ae865787ebf266c9d011f2578567ed913a7cb6c0ed899", + "proof/region/v1/arb/evaluator/interval.h": "f9d7416059d4b09979c22e6823a747f252c576558c750fe3e2ff92509894c7b3", + "proof/region/v1/arb/evaluator/main.c": "e9a3fa6b70b3a25eb6d6cf7eaba9a98d2fbe5cb7fdd3c1790219efb7fe20918d", + "proof/region/v1/arb/evaluator/region.c": "0026d501077911eae58933487a4cac0a83003cd70d1dbf0966890c29bfff8f99", + "proof/region/v1/arb/evaluator/region.h": "95da5117bb162c707b441242637d5e0e1bbeef2532ac1f10248f2b93ab16dcc8", + "proof/region/v1/arb/evaluator/wire.c": "4edb1120a8274774b8790eceea877c664f599bb9e039b0aa6e6ba8dafe124d47", + "proof/region/v1/arb/evaluator/wire.h": "bdf2ce9be9fce95a38c61e923b45038efb7bfab78842e38296114f0e83266c98", +} + +REQUIRED_BUILD_SOURCE_MODES_V1 = tuple( + (path, 0o755 if path == BUILD_RECIPE_PATH_V1 else 0o644) + for path in sorted(_PINNED_BUILD_SOURCE_SHA256_V1) +) + +BUILD_STDOUT_LIMIT_V1 = 16 * 1024 * 1024 +BUILD_STDERR_LIMIT_V1 = 16 * 1024 * 1024 +BUILD_TIMEOUT_NS_V1 = 2 * 60 * 60 * 1_000_000_000 +DOCKER_PROBE_OUTPUT_LIMIT_V1 = 1024 * 1024 +DOCKER_PROBE_TIMEOUT_NS_V1 = 30 * 1_000_000_000 +MAX_BUILD_SOURCE_FILE_BYTES_V1 = 16 * 1024 * 1024 +MAX_BUILD_SOURCE_TOTAL_BYTES_V1 = 32 * 1024 * 1024 + +# FLINT's exact locked qsieve path uses /tmp directly rather than TMPDIR. A +# container-private tmpfs preserves a read-only root without a host bind, +# volume, or reusable writable-layer scratch. POSIX sticky-directory mode is +# required because the container runs as the unprivileged host runner identity. +_BUILD_TMPFS_SPEC_V1 = "/tmp:rw,noexec,nosuid,nodev,mode=1777" + +_BUILD_SOURCES_ID_LABEL_V1 = b"labcolors.proof-region.arb-build-sources.v1\0" +_BUILD_INPUT_ID_LABEL_V1 = b"labcolors.proof-region.arb-compiler-inputs.v1\0" +_FORMULA_SUPPORT_ID_LABEL_V1 = b"labcolors.proof-region.arb-formula-support.v1\0" +_FLINT_COMMIT_CONTENT_ID_LABEL_V1 = ( + b"labcolors.proof-region.flint-commit-content.v1\0" +) +_FLINT_RELEASE_ONLY_ID_LABEL_V1 = ( + b"labcolors.proof-region.flint-project-pinned-release-only.v1\0" +) +_PIPELINE_POLICY_ID_LABEL_V1 = b"labcolors.proof-region.arb-pipeline-policy.v1\0" +_INVOCATION_ID_LABEL_V1 = b"labcolors.proof-region.arb-invocation.v1\0" +_PLATFORM_ID_LABEL_V1 = b"labcolors.proof-region.arb-run-platform.v1\0" +_BUILD_SOURCES_TOKEN = object() +_COMPARATOR_TOKEN = object() +_BUILD_OBSERVATION_TOKEN = object() +_PIPELINE_OBSERVATION_TOKEN = object() + + +def _blob(value: bytes) -> bytes: + return len(value).to_bytes(8, "big") + value + + +def _identity(label: bytes, chunks: tuple[bytes, ...]) -> bytes: + payload = b"".join(_blob(chunk) for chunk in chunks) + return hashlib.sha256(label + len(payload).to_bytes(8, "big") + payload).digest() + + +def _valid_digest(value: object) -> bool: + return type(value) is bytes and len(value) == 32 and value != bytes(32) + + +class BuildSourceReasonV1(StrEnum): + WRONG_TYPE = "wrong_type" + INVALID_PATH = "invalid_path" + INVALID_MODE = "invalid_mode" + INVALID_CONTENT = "invalid_content" + NONCANONICAL_SET = "noncanonical_set" + CONTENT_DRIFT = "content_drift" + + +@dataclass(frozen=True) +class BuildSourceAdmissionErrorV1(ValueError): + reason: BuildSourceReasonV1 + path: str + + def __str__(self) -> str: + return f"{self.reason.value}: {self.path}" + + +def _source_fail(reason: BuildSourceReasonV1, path: str) -> NoReturn: + raise BuildSourceAdmissionErrorV1(reason, path) + + +def _logical_path(value: object) -> str: + if type(value) is not str or not value or value.startswith("/") or "\\" in value: + _source_fail(BuildSourceReasonV1.INVALID_PATH, str(value)) + try: + encoded = value.encode("ascii") + except UnicodeEncodeError: + _source_fail(BuildSourceReasonV1.INVALID_PATH, value) + if ( + len(encoded) > 4096 + or any(byte < 0x20 or byte == 0x7F for byte in encoded) + or any(part in ("", ".", "..") for part in value.split("/")) + ): + _source_fail(BuildSourceReasonV1.INVALID_PATH, value) + return value + + +@dataclass(frozen=True) +class BuildSourceFileV1: + path: str + mode: int + contents: bytes + + def __post_init__(self) -> None: + _logical_path(self.path) + if type(self.mode) is not int or self.mode not in (0o644, 0o755): + _source_fail(BuildSourceReasonV1.INVALID_MODE, self.path) + if ( + type(self.contents) is not bytes + or not self.contents + or len(self.contents) > MAX_BUILD_SOURCE_FILE_BYTES_V1 + ): + _source_fail(BuildSourceReasonV1.INVALID_CONTENT, self.path) + + +@dataclass(frozen=True, init=False) +class AdmittedBuildSourcesV1: + """Owned exact local build-support closure. + + ``build_input_identity`` covers the recipe, generated C and evaluator files + named by that recipe. ``formula_support_identity`` separately covers the + formula spec, generator and generated C. The latter is support/replay + material; it does not claim that build.sh executed the generator. + """ + + files: tuple[BuildSourceFileV1, ...] + identity: bytes + + def __init__( + self, + files_value: tuple[BuildSourceFileV1, ...], + identity: bytes, + *, + _token: object, + ) -> None: + if _token is not _BUILD_SOURCES_TOKEN: + raise TypeError("AdmittedBuildSourcesV1 is created only by source admission") + if type(files_value) is not tuple or any( + type(item) is not BuildSourceFileV1 for item in files_value + ): + raise TypeError("invalid build source files") + if not _valid_digest(identity): + raise TypeError("invalid build source identity") + object.__setattr__(self, "files", files_value) + object.__setattr__(self, "identity", identity) + + def contents(self, path: str) -> bytes: + for item in self.files: + if item.path == path: + return item.contents + raise KeyError(path) + + @property + def formula_spec(self) -> bytes: + return self.contents(FORMULA_SPEC_PATH_V1) + + @property + def generated_formula(self) -> bytes: + return self.contents(GENERATED_FORMULA_PATH_V1) + + @cached_property + def build_input_identity(self) -> bytes: + direct = tuple( + item + for item in self.files + if item.path not in (FORMULA_SPEC_PATH_V1, FORMULA_GENERATOR_PATH_V1) + ) + return _source_subset_identity(_BUILD_INPUT_ID_LABEL_V1, direct) + + @cached_property + def formula_support_identity(self) -> bytes: + support_paths = frozenset( + ( + FORMULA_SPEC_PATH_V1, + FORMULA_GENERATOR_PATH_V1, + GENERATED_FORMULA_PATH_V1, + ) + ) + support = tuple(item for item in self.files if item.path in support_paths) + return _source_subset_identity(_FORMULA_SUPPORT_ID_LABEL_V1, support) + + +def _source_subset_identity( + label: bytes, + files_value: tuple[BuildSourceFileV1, ...], +) -> bytes: + chunks: list[bytes] = [len(files_value).to_bytes(4, "big")] + for item in files_value: + chunks.extend( + ( + item.path.encode("ascii"), + item.mode.to_bytes(4, "big"), + hashlib.sha256(item.contents).digest(), + len(item.contents).to_bytes(8, "big"), + ) + ) + return _identity(label, tuple(chunks)) + + +def _build_sources_identity(files_value: tuple[BuildSourceFileV1, ...]) -> bytes: + return _source_subset_identity(_BUILD_SOURCES_ID_LABEL_V1, files_value) + + +def admit_build_sources_v1( + files_value: tuple[BuildSourceFileV1, ...], +) -> AdmittedBuildSourcesV1: + if type(files_value) is not tuple or any( + type(item) is not BuildSourceFileV1 for item in files_value + ): + _source_fail(BuildSourceReasonV1.WRONG_TYPE, "files") + actual = tuple((item.path, item.mode) for item in files_value) + if actual != REQUIRED_BUILD_SOURCE_MODES_V1: + _source_fail(BuildSourceReasonV1.NONCANONICAL_SET, "files") + if sum(len(item.contents) for item in files_value) > MAX_BUILD_SOURCE_TOTAL_BYTES_V1: + _source_fail(BuildSourceReasonV1.INVALID_CONTENT, "files") + for item in files_value: + if hashlib.sha256(item.contents).hexdigest() != _PINNED_BUILD_SOURCE_SHA256_V1[item.path]: + _source_fail(BuildSourceReasonV1.CONTENT_DRIFT, item.path) + return AdmittedBuildSourcesV1( + files_value, + _build_sources_identity(files_value), + _token=_BUILD_SOURCES_TOKEN, + ) + + +class HostTrustBoundaryV1(StrEnum): + UNSEALED_LINUX_X64_DOCKER_HOST = "unsealed-linux-x64-docker-host" + + +def pipeline_policy_identity_v1( + host_trust: HostTrustBoundaryV1, +) -> bytes: + if type(host_trust) is not HostTrustBoundaryV1: + raise TypeError("host_trust must be HostTrustBoundaryV1") + return _identity( + _PIPELINE_POLICY_ID_LABEL_V1, + ( + OCI_IMAGE_REFERENCE_V1.encode("ascii"), + OCI_PLATFORM_V1.encode("ascii"), + host_trust.value.encode("ascii"), + b"build-observation=diagnostic-unsealed-v1", + b"run-observation=diagnostic-unsealed-v1", + b"network=none", + b"rootfs=readonly", + b"scratch-tmpfs=" + _BUILD_TMPFS_SPEC_V1.encode("ascii"), + b"cap-drop=all", + b"no-new-privileges=true", + b"inputs=readonly-bind", + b"workspace=readonly-bind", + f"source-snapshot-mtime-ns={snapshot.SOURCE_SNAPSHOT_MTIME_NS_V1}".encode( + "ascii" + ), + b"fresh-container-count=2", + ), + ) + + +class PipelineInputReasonV1(StrEnum): + WRONG_TYPE = "wrong_type" + FOREIGN_SOURCE_CAPABILITY = "foreign_source_capability" + FORMULA_MISMATCH = "formula_mismatch" + EXECUTION_LIMIT_MISMATCH = "execution_limit_mismatch" + + +@dataclass(frozen=True) +class PipelineInputErrorV1(ValueError): + reason: PipelineInputReasonV1 + field: str + + def __str__(self) -> str: + return f"{self.reason.value}: {self.field}" + + +@dataclass(frozen=True) +class FlintSourceContentPartitionV1: + """Structural FLINT archive partition, not an origin assertion. + + ``commit_content`` names the side which the project lock expects a future + authority to relate to the exact Git tree. ``project_pinned_release_only`` + names the separate release bytes consumed by the build. Neither identity + claims that those release-only bytes were generated from the commit. + """ + + commit_content_identity: bytes + commit_content_file_count: int + project_pinned_release_only_identity: bytes + project_pinned_release_only_file_count: int + + def __post_init__(self) -> None: + if not _valid_digest(self.commit_content_identity): + raise TypeError("invalid FLINT commit-content identity") + if not _valid_digest(self.project_pinned_release_only_identity): + raise TypeError("invalid FLINT project-pinned release-only identity") + if ( + type(self.commit_content_file_count) is not int + or self.commit_content_file_count <= 0 + or type(self.project_pinned_release_only_file_count) is not int + or self.project_pinned_release_only_file_count <= 0 + ): + raise TypeError("FLINT source partition must be nonempty on both sides") + + +def _archive_file_subset_identity( + label: bytes, + files_value: tuple[provenance.ArchiveFileV1, ...], +) -> bytes: + chunks: list[bytes] = [len(files_value).to_bytes(8, "big")] + for item in files_value: + chunks.extend( + ( + item.path.encode("ascii"), + item.mode.to_bytes(4, "big"), + item.length.to_bytes(8, "big"), + item.sha256, + ) + ) + return _identity(label, tuple(chunks)) + + +def _require_bound_source_capability_v1( + source_lock: provenance.ArbSourceLockV1, + admitted_sources: provenance.AdmittedArbSourcesV1, +) -> None: + if source_lock.identity != admitted_sources.source_lock_identity: + raise PipelineInputErrorV1( + PipelineInputReasonV1.FOREIGN_SOURCE_CAPABILITY, + "admitted_sources", + ) + for lock, admitted in zip( + source_lock.sources, + admitted_sources.sources, + strict=True, + ): + if lock.identity != admitted.source_lock_identity: + raise PipelineInputErrorV1( + PipelineInputReasonV1.FOREIGN_SOURCE_CAPABILITY, + "admitted_sources", + ) + + +def flint_source_content_partition_v1( + source_lock: provenance.ArbSourceLockV1, + admitted_sources: provenance.AdmittedArbSourcesV1, +) -> FlintSourceContentPartitionV1: + if type(source_lock) is not provenance.ArbSourceLockV1: + raise PipelineInputErrorV1(PipelineInputReasonV1.WRONG_TYPE, "source_lock") + if type(admitted_sources) is not provenance.AdmittedArbSourcesV1: + raise PipelineInputErrorV1( + PipelineInputReasonV1.WRONG_TYPE, + "admitted_sources", + ) + _require_bound_source_capability_v1(source_lock, admitted_sources) + + flint_lock = source_lock.sources[2] + flint_source = admitted_sources.sources[2] + if type(flint_lock.integrity) is not provenance.GitContentRelationPolicyV1: + raise PipelineInputErrorV1(PipelineInputReasonV1.WRONG_TYPE, "source_lock") + release_only_by_path = { + item.path: item + for item in flint_lock.integrity.project_pinned_release_only_files + } + release_only = tuple( + item for item in flint_source.files if item.path in release_only_by_path + ) + commit_content = tuple( + item for item in flint_source.files if item.path not in release_only_by_path + ) + if ( + len(commit_content) != flint_lock.integrity.common_file_count + or len(release_only) != len(release_only_by_path) + or any( + item.mode != release_only_by_path[item.path].mode + or item.length != release_only_by_path[item.path].length + or item.sha256 != release_only_by_path[item.path].sha256 + for item in release_only + ) + or len(commit_content) + len(release_only) != len(flint_source.files) + ): + raise PipelineInputErrorV1( + PipelineInputReasonV1.FOREIGN_SOURCE_CAPABILITY, + "flint_source_partition", + ) + return FlintSourceContentPartitionV1( + _archive_file_subset_identity( + _FLINT_COMMIT_CONTENT_ID_LABEL_V1, + commit_content, + ), + len(commit_content), + _archive_file_subset_identity( + _FLINT_RELEASE_ONLY_ID_LABEL_V1, + release_only, + ), + len(release_only), + ) + + +def _comparator_preimage_v1(label: bytes, chunks: tuple[bytes, ...]) -> bytes: + """Encode one independently versioned, ordered comparator preimage.""" + + if ( + type(label) is not bytes + or not label.startswith(b"labcolors.proof-region.arb-comparator.") + or not label.endswith(b".v1\0") + or type(chunks) is not tuple + or not chunks + or any(type(chunk) is not bytes for chunk in chunks) + ): + raise TypeError("invalid comparator preimage coordinates") + return label + b"\x01" + len(chunks).to_bytes(4, "big") + b"".join( + _blob(chunk) for chunk in chunks + ) + + +def _encoded_build_file_set_v1( + label: bytes, + files_value: tuple[BuildSourceFileV1, ...], +) -> bytes: + chunks: list[bytes] = [len(files_value).to_bytes(4, "big")] + for item in files_value: + chunks.extend( + ( + item.path.encode("ascii"), + item.mode.to_bytes(4, "big"), + len(item.contents).to_bytes(8, "big"), + item.contents, + ) + ) + return _comparator_preimage_v1(label, tuple(chunks)) + + +def _operation_allowlist_preimage_v1(formula_spec: bytes) -> bytes: + """Bind the exact ordered SSA operator contract from the admitted formula.""" + + if type(formula_spec) is not bytes or not formula_spec: + raise TypeError("formula_spec must be nonempty bytes") + lines = formula_spec.splitlines() + declarations: tuple[bytes, ...] | None = None + for index, line in enumerate(lines): + if not line.startswith(b"operators "): + continue + pieces = line.split(b" ") + if len(pieces) != 2 or not pieces[1].isdigit(): + raise ValueError("invalid formula operator count") + count = int(pieces[1]) + candidate = tuple(lines[index + 1 : index + 1 + count]) + if ( + count <= 0 + or len(candidate) != count + or any(not item.startswith(b"operator ") for item in candidate) + or ( + index + 1 + count < len(lines) + and lines[index + 1 + count].startswith(b"operator ") + ) + ): + raise ValueError("formula operator declarations do not match their count") + declarations = candidate + break + if declarations is None: + raise ValueError("formula has no operator contract") + return _comparator_preimage_v1( + b"labcolors.proof-region.arb-comparator.operation-allowlist.v1\0", + ( + b"exact-real-ssa-operator-declarations", + len(declarations).to_bytes(4, "big"), + *declarations, + ), + ) + + +@dataclass(frozen=True) +class ArbComparatorPreimagesV1: + engine_release: bytes + upstream_source: bytes + arithmetic_input_set: bytes + wrapper_source: bytes + evaluator_source: bytes + build_identity: bytes + operation_allowlist: bytes + test_observation: bytes + legal_file_set: bytes + exclusions: bytes + + def __post_init__(self) -> None: + values = tuple(getattr(self, item.name) for item in fields(self)) + if any(type(value) is not bytes or not value for value in values): + raise TypeError("comparator preimages must be nonempty exact bytes") + if len(set(values)) != len(values): + raise TypeError("comparator preimages must be independently domain-separated") + + +@dataclass(frozen=True, init=False) +class DiagnosticArbComparatorV1: + """Manifest declaration derived from admitted inputs and diagnostic BUILD.""" + + preimages: ArbComparatorPreimagesV1 + manifest: protocol.ContentResolvedComparatorManifestV2 + structural_source_identity: bytes + build_input_identity: bytes + pipeline_policy_identity: bytes + binary_sha256: bytes + rebuild_sha256s: tuple[bytes, bytes] + + def __new__(cls, *args: object, **kwargs: object) -> "DiagnosticArbComparatorV1": + if kwargs.get("_token") is not _COMPARATOR_TOKEN: + raise TypeError("DiagnosticArbComparatorV1 is controller-derived") + return object.__new__(cls) + + def __init__( + self, + preimages: ArbComparatorPreimagesV1, + manifest: protocol.ContentResolvedComparatorManifestV2, + structural_source_identity: bytes, + build_input_identity: bytes, + pipeline_policy_identity: bytes, + binary_sha256: bytes, + rebuild_sha256s: tuple[bytes, bytes], + *, + _token: object, + ) -> None: + if _token is not _COMPARATOR_TOKEN: + raise TypeError("DiagnosticArbComparatorV1 is controller-derived") + if type(preimages) is not ArbComparatorPreimagesV1: + raise TypeError("invalid comparator preimages") + if ( + type(manifest) is not protocol.ContentResolvedComparatorManifestV2 + or manifest.manifest.kind is not protocol.ComparatorKindV1.ARB + ): + raise TypeError("invalid Arb comparator manifest") + manifest_names = tuple( + item.name for item in fields(manifest.manifest) if item.name != "kind" + ) + preimage_names = tuple(item.name for item in fields(preimages)) + if manifest_names != preimage_names: + raise TypeError("comparator manifest/preimage schema drift") + by_digest = { + hashlib.sha256(getattr(preimages, name)).digest(): getattr(preimages, name) + for name in preimage_names + } + replayed = protocol.ContentResolvedComparatorManifestV2.admit( + manifest.manifest, + by_digest.get, + ) + if replayed.identity != manifest.identity: + raise TypeError("comparator manifest replay drift") + for name, value in ( + ("structural_source_identity", structural_source_identity), + ("build_input_identity", build_input_identity), + ("pipeline_policy_identity", pipeline_policy_identity), + ("binary_sha256", binary_sha256), + ): + if not _valid_digest(value): + raise TypeError(f"invalid {name}") + if ( + type(rebuild_sha256s) is not tuple + or rebuild_sha256s != (binary_sha256, binary_sha256) + ): + raise TypeError("invalid comparator rebuild binding") + for field_name, field_value in ( + ("preimages", preimages), + ("manifest", manifest), + ("structural_source_identity", structural_source_identity), + ("build_input_identity", build_input_identity), + ("pipeline_policy_identity", pipeline_policy_identity), + ("binary_sha256", binary_sha256), + ("rebuild_sha256s", rebuild_sha256s), + ): + object.__setattr__(self, field_name, field_value) + + @property + def identity(self) -> bytes: + return self.manifest.identity + + +@dataclass(frozen=True) +class PipelineRequestV1: + source_lock: provenance.ArbSourceLockV1 + admitted_sources: provenance.AdmittedArbSourcesV1 + build_sources: AdmittedBuildSourcesV1 + job: protocol.ProofJobV1 + execution_limits: executor.ExecutionLimitsV1 + host_trust: HostTrustBoundaryV1 + + def __post_init__(self) -> None: + expected_types = ( + ("source_lock", self.source_lock, provenance.ArbSourceLockV1), + ("admitted_sources", self.admitted_sources, provenance.AdmittedArbSourcesV1), + ("build_sources", self.build_sources, AdmittedBuildSourcesV1), + ("job", self.job, protocol.ProofJobV1), + ("execution_limits", self.execution_limits, executor.ExecutionLimitsV1), + ("host_trust", self.host_trust, HostTrustBoundaryV1), + ) + for field_name, value, expected_type in expected_types: + if type(value) is not expected_type: + raise PipelineInputErrorV1(PipelineInputReasonV1.WRONG_TYPE, field_name) + _require_bound_source_capability_v1(self.source_lock, self.admitted_sources) + flint_source_content_partition_v1(self.source_lock, self.admitted_sources) + if self.build_sources.formula_spec != self.job.formula_spec: + raise PipelineInputErrorV1(PipelineInputReasonV1.FORMULA_MISMATCH, "job") + job_bytes = self.job.encode() + invocation_bytes = sum( + len(value) + 1 + for value in ( + b"arb-evaluator", + b"--manifest-identity", + bytes(32).hex().encode("ascii"), + b"--job", + b"/dev/stdin", + ) + ) + sum( + len(key) + len(value) + 2 + for key, value in ((b"LC_ALL", b"C"), (b"TZ", b"UTC")) + ) + if ( + len(job_bytes) > self.execution_limits.max_stdin_bytes + or invocation_bytes > self.execution_limits.max_argument_bytes + ): + raise PipelineInputErrorV1( + PipelineInputReasonV1.EXECUTION_LIMIT_MISMATCH, + "execution_limits", + ) + + +class DockerBlockerReasonV1(StrEnum): + HOST_NOT_LINUX_AMD64 = "host_not_linux_amd64" + DOCKER_UNAVAILABLE = "docker_unavailable" + IMAGE_UNAVAILABLE = "image_unavailable" + IMAGE_IDENTITY_MISMATCH = "image_identity_mismatch" + ISOLATION_UNAVAILABLE = "isolation_unavailable" + SAME_OBJECT_OUTPUT_UNAVAILABLE = "same_object_output_unavailable" + BACKEND_CONTRACT = "backend_contract" + + +@dataclass(frozen=True) +class DockerUnsupportedV1: + reason: DockerBlockerReasonV1 + detail: str + + def __post_init__(self) -> None: + if type(self.reason) is not DockerBlockerReasonV1: + raise TypeError("invalid Docker blocker reason") + if type(self.detail) is not str or not self.detail or len(self.detail) > 4096: + raise TypeError("invalid Docker blocker detail") + + +@dataclass(frozen=True) +class DockerSupportedV1: + image_reference: str + platform: str + daemon_observation_sha256: bytes + + def __post_init__(self) -> None: + if self.image_reference != OCI_IMAGE_REFERENCE_V1: + raise TypeError("wrong OCI image reference") + if self.platform != OCI_PLATFORM_V1: + raise TypeError("wrong OCI platform") + if not _valid_digest(self.daemon_observation_sha256): + raise TypeError("invalid Docker daemon observation digest") + + +DockerCapabilityReportV1: TypeAlias = DockerSupportedV1 | DockerUnsupportedV1 + + +def _absolute_path(value: object, field_name: str) -> Path: + if not isinstance(value, Path) or not value.is_absolute(): + raise TypeError(f"{field_name} must be an absolute Path") + if any(character in str(value) for character in (",", "\n", "\r", "\0")): + raise TypeError(f"{field_name} is not Docker-mount-safe") + return value + + +_CONTAINER_NAME_PREFIX_V1 = "labcolors-arb-build-v1-" + + +def _container_name(value: object) -> str: + if ( + type(value) is not str + or not value.startswith(_CONTAINER_NAME_PREFIX_V1) + or len(value) > 128 + or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in value) + ): + raise TypeError("invalid controller-owned Docker container name") + return value + + +@dataclass(frozen=True) +class DockerBuildRequestV1: + attempt: int + root_directory: Path + inputs_directory: Path + workspace_directory: Path + build_directory: Path + output_directory: Path + cid_file: Path + container_name: str + + def __post_init__(self) -> None: + if type(self.attempt) is not int or self.attempt not in (1, 2): + raise TypeError("attempt must be 1 or 2") + paths = tuple( + _absolute_path(getattr(self, field_name), field_name) + for field_name in ( + "root_directory", + "inputs_directory", + "workspace_directory", + "build_directory", + "output_directory", + "cid_file", + ) + ) + _container_name(self.container_name) + root = self.root_directory + if len(set(paths)) != len(paths): + raise TypeError("build paths must be distinct") + for path in paths[1:]: + try: + path.relative_to(root) + except ValueError: + raise TypeError("build path escapes controller root") from None + + +def _bounded_bytes(value: object, maximum: int, field_name: str) -> bytes: + if type(value) is not bytes or len(value) > maximum: + raise TypeError(f"invalid {field_name}") + return value + + +@dataclass(frozen=True) +class DockerBuildExitedV1: + returncode: int + stdout: bytes + stderr: bytes + + def __post_init__(self) -> None: + if type(self.returncode) is not int: + raise TypeError("invalid Docker returncode") + _bounded_bytes(self.stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(self.stderr, BUILD_STDERR_LIMIT_V1, "stderr") + + +@dataclass(frozen=True) +class DockerBuildTimedOutV1: + stdout: bytes + stderr: bytes + + def __post_init__(self) -> None: + _bounded_bytes(self.stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(self.stderr, BUILD_STDERR_LIMIT_V1, "stderr") + + +class DockerOutputStreamV1(StrEnum): + STDOUT = "stdout" + STDERR = "stderr" + + +@dataclass(frozen=True) +class DockerBuildOutputLimitV1: + stream: DockerOutputStreamV1 + stdout: bytes + stderr: bytes + + def __post_init__(self) -> None: + if type(self.stream) is not DockerOutputStreamV1: + raise TypeError("invalid Docker output stream") + _bounded_bytes(self.stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(self.stderr, BUILD_STDERR_LIMIT_V1, "stderr") + + +@dataclass(frozen=True) +class DockerBuildObserverFailureV1: + detail: str + + def __post_init__(self) -> None: + if type(self.detail) is not str or not self.detail or len(self.detail) > 4096: + raise TypeError("invalid Docker observer failure") + + +class DockerCleanupTriggerV1(StrEnum): + PROCESS_EXIT = "process_exit" + TIMEOUT = "timeout" + OUTPUT_LIMIT = "output_limit" + OBSERVER_FAILURE = "observer_failure" + + +@dataclass(frozen=True) +class DockerBuildCleanupFailureV1: + trigger: DockerCleanupTriggerV1 + detail: str + stdout: bytes + stderr: bytes + + def __post_init__(self) -> None: + if type(self.trigger) is not DockerCleanupTriggerV1: + raise TypeError("invalid Docker cleanup trigger") + if type(self.detail) is not str or not self.detail or len(self.detail) > 4096: + raise TypeError("invalid Docker cleanup failure") + _bounded_bytes(self.stdout, BUILD_STDOUT_LIMIT_V1, "stdout") + _bounded_bytes(self.stderr, BUILD_STDERR_LIMIT_V1, "stderr") + + +DockerBuildProcessObservationV1: TypeAlias = ( + DockerBuildExitedV1 + | DockerBuildTimedOutV1 + | DockerBuildOutputLimitV1 + | DockerBuildObserverFailureV1 + | DockerBuildCleanupFailureV1 +) + + +class DockerBuildBackendV1(Protocol): + def probe(self) -> DockerCapabilityReportV1: ... + + def run_build( + self, + request: DockerBuildRequestV1, + ) -> DockerBuildProcessObservationV1: ... + + +def _archive_file_manifest_bytes_v1( + files_value: tuple[provenance.ArchiveFileV1, ...], +) -> bytes: + chunks: list[bytes] = [len(files_value).to_bytes(8, "big")] + for item in files_value: + chunks.extend( + ( + item.path.encode("ascii"), + item.mode.to_bytes(4, "big"), + item.length.to_bytes(8, "big"), + item.sha256, + ) + ) + return b"".join(_blob(chunk) for chunk in chunks) + + +def _source_snapshot_chunks_v1( + lock: provenance.SourceReleaseLockV1, + source: provenance.SafeSourceArchiveV1, +) -> tuple[bytes, ...]: + return ( + bytes((int(lock.role),)), + lock.encode(), + source.source_lock_identity, + source.archive_sha256, + source.tree_identity, + source.regular_file_count.to_bytes(8, "big"), + source.regular_file_bytes.to_bytes(8, "big"), + _archive_file_manifest_bytes_v1(source.files), + len(source.archive_bytes).to_bytes(8, "big"), + hashlib.sha256(source.archive_bytes).digest(), + ) + + +def _build_process_bytes_v1(process: DockerBuildExitedV1) -> bytes: + if type(process) is not DockerBuildExitedV1: + raise TypeError("only successful typed build observations are encodable") + return b"".join( + ( + process.returncode.to_bytes(4, "big", signed=True), + len(process.stdout).to_bytes(8, "big"), + hashlib.sha256(process.stdout).digest(), + len(process.stderr).to_bytes(8, "big"), + hashlib.sha256(process.stderr).digest(), + ) + ) + + +def _derive_arb_comparator_for_build_v1( + request: PipelineRequestV1, + docker_report: DockerSupportedV1, + binary: bytes, + rebuild_sha256s: tuple[bytes, bytes], + build_processes: tuple[DockerBuildExitedV1, DockerBuildExitedV1], +) -> DiagnosticArbComparatorV1: + """Derive all ten coordinates without accepting a caller digest/resolver.""" + + if type(request) is not PipelineRequestV1: + raise TypeError("request must be PipelineRequestV1") + if type(docker_report) is not DockerSupportedV1: + raise TypeError("docker_report must be DockerSupportedV1") + if type(binary) is not bytes or not binary: + raise TypeError("binary must be exact nonempty bytes") + binary_sha256 = hashlib.sha256(binary).digest() + if ( + type(build_processes) is not tuple + or len(build_processes) != 2 + or any(type(item) is not DockerBuildExitedV1 for item in build_processes) + or any(item.returncode != 0 for item in build_processes) + or rebuild_sha256s != (binary_sha256, binary_sha256) + ): + raise TypeError("comparator derivation requires two equal successful builds") + pipeline_policy_identity = pipeline_policy_identity_v1(request.host_trust) + flint_lock = request.source_lock.sources[2] + flint_source = request.admitted_sources.sources[2] + if type(flint_lock.integrity) is not provenance.GitContentRelationPolicyV1: + raise TypeError("FLINT requires the exact content-relation policy") + + exclusions = _comparator_preimage_v1( + b"labcolors.proof-region.arb-comparator.exclusions.v1\0", + ( + b"gap:host-and-docker-daemon-not-source-bound", + b"gap:unsealed-diagnostic-build-observer", + b"gap:unsealed-diagnostic-run-observer", + b"gap:libc-libm-libpthread-libgcc-and-build-utility-source", + b"gap:no-per-test-result-records", + b"gap:no-git-derivation-for-project-pinned-release-only-files", + b"gap:no-origin-authority-reverification", + request.host_trust.value.encode("ascii"), + b"build-observation=diagnostic-unsealed-v1", + b"run-observation=diagnostic-unsealed-v1", + len(flint_lock.integrity.omitted_paths).to_bytes(4, "big"), + *( + path.encode("ascii") + for path in flint_lock.integrity.omitted_paths + ), + len( + flint_lock.integrity.project_pinned_release_only_files + ).to_bytes(4, "big"), + *( + item.encode() + for item in flint_lock.integrity.project_pinned_release_only_files + ), + ), + ) + + upstream_chunks: list[bytes] = [ + request.source_lock.encode(), + request.admitted_sources.source_lock_identity, + len(request.source_lock.sources).to_bytes(4, "big"), + ] + for lock, source in zip( + request.source_lock.sources, + request.admitted_sources.sources, + strict=True, + ): + upstream_chunks.extend(_source_snapshot_chunks_v1(lock, source)) + upstream_source = _comparator_preimage_v1( + b"labcolors.proof-region.arb-comparator.upstream-source.v1\0", + tuple(upstream_chunks), + ) + + operation_allowlist = _operation_allowlist_preimage_v1( + request.build_sources.formula_spec + ) + arithmetic_chunks: list[bytes] = [ + b"exact admitted GMP MPFR FLINT source snapshots and pinned static-build boundary", + len(request.source_lock.sources).to_bytes(4, "big"), + ] + for lock, source in zip( + request.source_lock.sources, + request.admitted_sources.sources, + strict=True, + ): + arithmetic_chunks.extend( + ( + bytes((int(lock.role),)), + lock.identity, + source.archive_sha256, + source.tree_identity, + ) + ) + arithmetic_chunks.extend( + ( + OCI_IMAGE_REFERENCE_V1.encode("ascii"), + OCI_PLATFORM_V1.encode("ascii"), + hashlib.sha256(operation_allowlist).digest(), + hashlib.sha256(exclusions).digest(), + ) + ) + arithmetic_input_set = _comparator_preimage_v1( + b"labcolors.proof-region.arb-comparator.arithmetic-input-set.v1\0", + tuple(arithmetic_chunks), + ) + + wrapper_paths = frozenset( + ( + "proof/region/v1/arb/evaluator/formula.h", + "proof/region/v1/arb/evaluator/interval.c", + "proof/region/v1/arb/evaluator/interval.h", + ) + ) + wrapper_files = tuple( + item for item in request.build_sources.files if item.path in wrapper_paths + ) + evaluator_files = tuple( + item + for item in request.build_sources.files + if item.path not in ( + FORMULA_SPEC_PATH_V1, + FORMULA_GENERATOR_PATH_V1, + BUILD_RECIPE_PATH_V1, + ) + and item.path not in wrapper_paths + ) + wrapper_source = _encoded_build_file_set_v1( + b"labcolors.proof-region.arb-comparator.wrapper-source.v1\0", + wrapper_files, + ) + evaluator_source = _encoded_build_file_set_v1( + b"labcolors.proof-region.arb-comparator.evaluator-source.v1\0", + evaluator_files, + ) + + process_bytes = tuple(_build_process_bytes_v1(item) for item in build_processes) + build_identity = _comparator_preimage_v1( + b"labcolors.proof-region.arb-comparator.build-identity.v1\0", + ( + request.build_sources.contents(BUILD_RECIPE_PATH_V1), + request.build_sources.build_input_identity, + request.build_sources.formula_support_identity, + OCI_IMAGE_REFERENCE_V1.encode("ascii"), + OCI_PLATFORM_V1.encode("ascii"), + docker_report.daemon_observation_sha256, + pipeline_policy_identity, + b"build-observation=diagnostic-unsealed-v1", + len(build_processes).to_bytes(4, "big"), + *process_bytes, + binary_sha256, + rebuild_sha256s[0], + rebuild_sha256s[1], + len(binary).to_bytes(8, "big"), + binary_sha256, + ), + ) + test_observation = _comparator_preimage_v1( + b"labcolors.proof-region.arb-comparator.test-observation.v1\0", + ( + b"kind:aggregate-outer-process-observation-no-per-test-records", + request.build_sources.contents(BUILD_RECIPE_PATH_V1), + len(build_processes).to_bytes(4, "big"), + *process_bytes, + ), + ) + + legal_chunks: list[bytes] = [ + b"ordered admitted legal-file set; no legal-compliance claim", + len(request.source_lock.sources).to_bytes(4, "big"), + ] + for lock, source in zip( + request.source_lock.sources, + request.admitted_sources.sources, + strict=True, + ): + actual_by_path = {item.path: item for item in source.files} + legal_chunks.extend( + ( + bytes((int(lock.role),)), + lock.identity, + source.archive_sha256, + source.tree_identity, + len(lock.legal_files).to_bytes(4, "big"), + ) + ) + for declaration in lock.legal_files: + actual = actual_by_path.get(declaration.path) + if ( + actual is None + or actual.length != declaration.length + or actual.sha256 != declaration.sha256 + ): + raise TypeError("admitted legal-file set drift") + legal_chunks.extend( + ( + declaration.encode(), + actual.path.encode("ascii"), + actual.mode.to_bytes(4, "big"), + actual.length.to_bytes(8, "big"), + actual.sha256, + ) + ) + legal_file_set = _comparator_preimage_v1( + b"labcolors.proof-region.arb-comparator.legal-file-set.v1\0", + tuple(legal_chunks), + ) + + engine_release = _comparator_preimage_v1( + b"labcolors.proof-region.arb-comparator.engine-release.v1\0", + ( + b"FLINT release lock declaration", + flint_lock.encode(), + flint_source.source_lock_identity, + ), + ) + preimages = ArbComparatorPreimagesV1( + engine_release, + upstream_source, + arithmetic_input_set, + wrapper_source, + evaluator_source, + build_identity, + operation_allowlist, + test_observation, + legal_file_set, + exclusions, + ) + coordinates = tuple( + hashlib.sha256(getattr(preimages, item.name)).digest() + for item in fields(preimages) + ) + manifest_value = protocol.ComparatorManifestV2( + protocol.ComparatorKindV1.ARB, + *coordinates, + ) + by_digest = { + coordinate: getattr(preimages, item.name) + for coordinate, item in zip(coordinates, fields(preimages), strict=True) + } + resolved = protocol.ContentResolvedComparatorManifestV2.admit( + manifest_value, + by_digest.get, + ) + return DiagnosticArbComparatorV1( + preimages, + resolved, + request.admitted_sources.identity, + request.build_sources.build_input_identity, + pipeline_policy_identity, + binary_sha256, + rebuild_sha256s, + _token=_COMPARATOR_TOKEN, + ) + + +class NativeDockerBuildBackendV1: + """Docker adapter whose probe observes only Linux x64 and its daemon.""" + + def __init__( + self, + docker_path: Path, + *, + platform_name: str | None = None, + machine_name: str | None = None, + monotonic_ns: object = time.monotonic_ns, + ) -> None: + if not isinstance(docker_path, Path) or not docker_path.is_absolute(): + raise TypeError("docker_path must be an absolute Path") + self._docker_path = docker_path + self._platform_name = ( + platform.system().lower() if platform_name is None else platform_name + ) + self._machine_name = platform.machine() if machine_name is None else machine_name + self._monotonic_ns = monotonic_ns + + @staticmethod + def _environment() -> dict[str, str]: + return { + "HOME": "/nonexistent", + "PATH": "/usr/bin:/bin", + "DOCKER_CONFIG": "/nonexistent", + } + + def probe(self) -> DockerCapabilityReportV1: + if self._platform_name != "linux" or self._machine_name.lower() not in ( + "x86_64", + "amd64", + ): + return DockerUnsupportedV1( + DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64, + "controlled build requires a Linux amd64 Docker host", + ) + try: + metadata = self._docker_path.lstat() + except OSError: + return DockerUnsupportedV1( + DockerBlockerReasonV1.DOCKER_UNAVAILABLE, + "exact Docker CLI path is unavailable", + ) + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + return DockerUnsupportedV1( + DockerBlockerReasonV1.DOCKER_UNAVAILABLE, + "Docker CLI must be one regular non-symlink path", + ) + commands = ( + ( + str(self._docker_path), + "version", + "--format", + "{{json .Server}}", + ), + ( + str(self._docker_path), + "image", + "inspect", + OCI_IMAGE_REFERENCE_V1, + ), + ) + outputs: list[bytes] = [] + for index, command in enumerate(commands): + result = self._observe_command( + command, + stdout_limit=DOCKER_PROBE_OUTPUT_LIMIT_V1, + stderr_limit=DOCKER_PROBE_OUTPUT_LIMIT_V1, + timeout_ns=DOCKER_PROBE_TIMEOUT_NS_V1, + cid_file=None, + ) + if ( + type(result) is not DockerBuildExitedV1 + or result.returncode != 0 + or not result.stdout + or result.stderr + ): + return DockerUnsupportedV1( + DockerBlockerReasonV1.DOCKER_UNAVAILABLE + if index == 0 + else DockerBlockerReasonV1.IMAGE_UNAVAILABLE, + "Docker daemon probe failed" + if index == 0 + else "pinned image is not locally inspectable", + ) + outputs.append(result.stdout) + try: + inspected = json.loads(outputs[1]) + if type(inspected) is not list or len(inspected) != 1: + raise ValueError("wrong image inspection cardinality") + image = inspected[0] + if type(image) is not dict: + raise ValueError("wrong image inspection shape") + repo_digests = image.get("RepoDigests") + if ( + image.get("Os") != "linux" + or image.get("Architecture") not in ("amd64", "x86_64") + or type(repo_digests) is not list + or OCI_IMAGE_REFERENCE_V1 not in repo_digests + ): + raise ValueError("foreign image coordinate") + except (ValueError, TypeError, json.JSONDecodeError): + return DockerUnsupportedV1( + DockerBlockerReasonV1.IMAGE_IDENTITY_MISMATCH, + "local image does not match pinned linux/amd64 manifest", + ) + daemon_digest = _identity( + b"labcolors.proof-region.docker-daemon-observation.v1\0", + tuple(outputs), + ) + return DockerSupportedV1( + OCI_IMAGE_REFERENCE_V1, + OCI_PLATFORM_V1, + daemon_digest, + ) + + def command_for(self, request: DockerBuildRequestV1) -> tuple[str, ...]: + if type(request) is not DockerBuildRequestV1: + raise TypeError("request must be DockerBuildRequestV1") + mounts = ( + f"type=bind,src={request.inputs_directory},dst=/inputs,readonly,bind-propagation=private", + f"type=bind,src={request.workspace_directory},dst=/workspace,readonly,bind-propagation=private", + f"type=bind,src={request.build_directory},dst=/build,bind-propagation=private", + f"type=bind,src={request.output_directory},dst=/out,bind-propagation=private", + ) + command = [ + str(self._docker_path), + "run", + "--rm", + "--pull", + "never", + "--platform", + OCI_PLATFORM_V1, + "--network", + "none", + "--read-only", + "--tmpfs", + _BUILD_TMPFS_SPEC_V1, + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges:true", + "--name", + request.container_name, + "--hostname", + "labcolors-arb-build-v1", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--workdir", + "/", + "--cidfile", + str(request.cid_file), + ] + for mount in mounts: + command.extend(("--mount", mount)) + command.extend( + ( + "--entrypoint", + "/bin/sh", + OCI_IMAGE_REFERENCE_V1, + f"/workspace/{BUILD_RECIPE_PATH_V1}", + ) + ) + return tuple(command) + + def run_build( + self, + request: DockerBuildRequestV1, + ) -> DockerBuildProcessObservationV1: + if type(request) is not DockerBuildRequestV1: + raise TypeError("request must be DockerBuildRequestV1") + return self._observe_command( + self.command_for(request), + stdout_limit=BUILD_STDOUT_LIMIT_V1, + stderr_limit=BUILD_STDERR_LIMIT_V1, + timeout_ns=BUILD_TIMEOUT_NS_V1, + cid_file=request.cid_file, + container_name=request.container_name, + ) + + def _observe_command( + self, + command: tuple[str, ...], + *, + stdout_limit: int, + stderr_limit: int, + timeout_ns: int, + cid_file: Path | None, + container_name: str | None = None, + ) -> DockerBuildProcessObservationV1: + if ( + type(command) is not tuple + or not command + or any(type(item) is not str or not item or "\0" in item for item in command) + ): + raise TypeError("command must be a nonempty string tuple") + if ( + type(stdout_limit) is not int + or stdout_limit <= 0 + or stdout_limit > BUILD_STDOUT_LIMIT_V1 + or type(stderr_limit) is not int + or stderr_limit <= 0 + or stderr_limit > BUILD_STDERR_LIMIT_V1 + or type(timeout_ns) is not int + or timeout_ns <= 0 + or timeout_ns > BUILD_TIMEOUT_NS_V1 + ): + raise TypeError("invalid Docker observation limits") + if (cid_file is None) != (container_name is None): + raise TypeError("Docker cleanup requires both CID file and exact name") + if cid_file is not None: + _absolute_path(cid_file, "cid_file") + _container_name(container_name) + try: + process = subprocess.Popen( + command, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd="/", + env=self._environment(), + close_fds=True, + start_new_session=True, + ) + except OSError: + return DockerBuildObserverFailureV1("cannot start Docker CLI") + if process.stdout is None or process.stderr is None: + stop_detail = self._stop_process(process) + cleanup_detail = ( + self._cleanup_container(cid_file, container_name) + if cid_file is not None and container_name is not None + else None + ) + if stop_detail is not None or cleanup_detail is not None: + return DockerBuildCleanupFailureV1( + DockerCleanupTriggerV1.OBSERVER_FAILURE, + stop_detail or cleanup_detail or "Docker cleanup failed", + b"", + b"", + ) + return DockerBuildObserverFailureV1("Docker pipes unavailable") + + stdout = bytearray() + stderr = bytearray() + streams = { + process.stdout.fileno(): (DockerOutputStreamV1.STDOUT, stdout, stdout_limit), + process.stderr.fileno(): (DockerOutputStreamV1.STDERR, stderr, stderr_limit), + } + selector = selectors.DefaultSelector() + terminal: DockerOutputStreamV1 | None = None + timed_out = False + observer_failed = False + try: + for descriptor in streams: + os.set_blocking(descriptor, False) + selector.register(descriptor, selectors.EVENT_READ) + start = self._clock() + deadline = start + timeout_ns + while selector.get_map() or process.poll() is None: + now = self._clock() + if now >= deadline: + timed_out = True + break + timeout = min((deadline - now) / 1_000_000_000, 0.1) + for key, _events in selector.select(timeout): + stream, target, maximum = streams[key.fd] + try: + chunk = os.read(key.fd, min(64 * 1024, maximum + 1 - len(target))) + except BlockingIOError: + continue + if not chunk: + selector.unregister(key.fd) + continue + target.extend(chunk) + if len(target) > maximum: + del target[maximum:] + terminal = stream + break + if terminal is not None: + break + except Exception: + observer_failed = True + finally: + selector.close() + + stop_detail: str | None = None + if timed_out or terminal is not None or observer_failed: + stop_detail = self._stop_process(process) + else: + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + timed_out = True + stop_detail = self._stop_process(process) + process.stdout.close() + process.stderr.close() + cleanup_detail = ( + self._cleanup_container(cid_file, container_name) + if cid_file is not None and container_name is not None + else None + ) + if stop_detail is not None or cleanup_detail is not None: + trigger = DockerCleanupTriggerV1.PROCESS_EXIT + if observer_failed: + trigger = DockerCleanupTriggerV1.OBSERVER_FAILURE + elif terminal is not None: + trigger = DockerCleanupTriggerV1.OUTPUT_LIMIT + elif timed_out: + trigger = DockerCleanupTriggerV1.TIMEOUT + return DockerBuildCleanupFailureV1( + trigger, + stop_detail or cleanup_detail or "Docker cleanup failed", + bytes(stdout), + bytes(stderr), + ) + if observer_failed: + return DockerBuildObserverFailureV1("Docker output observation failed") + if terminal is not None: + return DockerBuildOutputLimitV1(terminal, bytes(stdout), bytes(stderr)) + if timed_out: + return DockerBuildTimedOutV1(bytes(stdout), bytes(stderr)) + if type(process.returncode) is not int: + return DockerBuildObserverFailureV1("Docker returncode unavailable") + return DockerBuildExitedV1(process.returncode, bytes(stdout), bytes(stderr)) + + def _clock(self) -> int: + value = self._monotonic_ns() + if type(value) is not int or value < 0: + raise RuntimeError("invalid monotonic clock") + return value + + def _stop_process( + self, + process: subprocess.Popen[bytes], + ) -> str | None: + failed = False + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + except OSError: + try: + process.kill() + except ProcessLookupError: + pass + except OSError: + failed = True + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + failed = True + if process.poll() is None: + failed = True + return "Docker CLI process could not be terminated" if failed else None + + @staticmethod + def _admitted_container_id(cid_file: Path) -> str | None: + try: + descriptor = os.open( + cid_file, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + except OSError: + return None + try: + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_nlink != 1 + or metadata.st_size not in (64, 65) + ): + return None + raw = os.read(descriptor, 66) + except OSError: + return None + finally: + os.close(descriptor) + if len(raw) == 65 and raw.endswith(b"\n"): + raw = raw[:-1] + if len(raw) != 64 or any( + byte not in b"0123456789abcdef" for byte in raw + ): + return None + return raw.decode("ascii") + + def _observe_cleanup_command( + self, + command: tuple[str, ...], + ) -> DockerBuildProcessObservationV1: + return self._observe_command( + command, + stdout_limit=DOCKER_PROBE_OUTPUT_LIMIT_V1, + stderr_limit=DOCKER_PROBE_OUTPUT_LIMIT_V1, + timeout_ns=DOCKER_PROBE_TIMEOUT_NS_V1, + cid_file=None, + ) + + def _cleanup_container(self, cid_file: Path, container_name: str) -> str | None: + _absolute_path(cid_file, "cid_file") + _container_name(container_name) + container_id = self._admitted_container_id(cid_file) + removal_coordinates = ( + (container_id, container_name) + if container_id is not None + else (container_name,) + ) + try: + for coordinate in removal_coordinates: + self._observe_cleanup_command( + ( + str(self._docker_path), + "container", + "rm", + "--force", + coordinate, + ) + ) + filters = [f"name=^/{container_name}$"] + if container_id is not None: + filters.append(f"id={container_id}") + for filter_value in filters: + observation = self._observe_cleanup_command( + ( + str(self._docker_path), + "container", + "ls", + "--all", + "--quiet", + "--no-trunc", + "--filter", + filter_value, + ) + ) + if ( + type(observation) is not DockerBuildExitedV1 + or observation.returncode != 0 + or observation.stdout + or observation.stderr + ): + return "Docker container absence could not be verified" + except Exception: + return "Docker container cleanup observer raised" + return None + + +class BuildFailureReasonV1(StrEnum): + BACKEND_CONTRACT = "backend_contract" + PROCESS_FAILED = "process_failed" + CLEANUP_FAILED = "cleanup_failed" + INPUT_CHANGED = "input_changed" + INVALID_OUTPUT = "invalid_output" + + +@dataclass(frozen=True) +class PipelineBlockedV1: + reason: DockerBlockerReasonV1 + detail: str + + +@dataclass(frozen=True) +class BuildRejectedV1: + attempt: int + reason: BuildFailureReasonV1 + process: DockerBuildProcessObservationV1 | None = None + + +@dataclass(frozen=True) +class NonReproducibleBuildV1: + first_sha256: bytes + second_sha256: bytes + + +class ExecutionFailureReasonV1(StrEnum): + UNSUPPORTED = "unsupported" + PROCESS_FAILED = "process_failed" + STDERR_NOT_EMPTY = "stderr_not_empty" + BINARY_MISMATCH = "binary_mismatch" + BACKEND_CONTRACT = "backend_contract" + + +@dataclass(frozen=True) +class ExecutionRejectedV1: + reason: ExecutionFailureReasonV1 + observation: object + + +class TranscriptFailureReasonV1(StrEnum): + INVALID_WIRE = "invalid_wire" + FOREIGN_BINDING = "foreign_binding" + + +@dataclass(frozen=True) +class TranscriptRejectedV1: + reason: TranscriptFailureReasonV1 + detail: str + + +@dataclass(frozen=True, init=False) +class DiagnosticBuildObservationV1: + """Controller-owned two-build observation with no native-evidence claim.""" + + structural_source_identity: bytes + flint_commit_content_identity: bytes + flint_commit_content_file_count: int + flint_project_pinned_release_only_identity: bytes + flint_project_pinned_release_only_file_count: int + build_input_identity: bytes + formula_support_identity: bytes + pipeline_policy_identity: bytes + docker_daemon_observation_sha256: bytes + oci_image_reference: str + oci_platform: str + binary_sha256: bytes + rebuild_sha256s: tuple[bytes, bytes] + host_trust: HostTrustBoundaryV1 + build_processes: tuple[DockerBuildExitedV1, DockerBuildExitedV1] + comparator: DiagnosticArbComparatorV1 + _binary: bytes + + def __init__( + self, + structural_source_identity: bytes, + flint_commit_content_identity: bytes, + flint_commit_content_file_count: int, + flint_project_pinned_release_only_identity: bytes, + flint_project_pinned_release_only_file_count: int, + build_input_identity: bytes, + formula_support_identity: bytes, + pipeline_policy_identity: bytes, + docker_daemon_observation_sha256: bytes, + oci_image_reference: str, + oci_platform: str, + binary_sha256: bytes, + rebuild_sha256s: tuple[bytes, bytes], + host_trust: HostTrustBoundaryV1, + build_processes: tuple[DockerBuildExitedV1, DockerBuildExitedV1], + comparator: DiagnosticArbComparatorV1, + binary: bytes, + *, + _token: object, + ) -> None: + if _token is not _BUILD_OBSERVATION_TOKEN: + raise TypeError("DiagnosticBuildObservationV1 is controller-only") + for name, value in ( + ("structural_source_identity", structural_source_identity), + ("flint_commit_content_identity", flint_commit_content_identity), + ( + "flint_project_pinned_release_only_identity", + flint_project_pinned_release_only_identity, + ), + ("build_input_identity", build_input_identity), + ("formula_support_identity", formula_support_identity), + ("pipeline_policy_identity", pipeline_policy_identity), + ("docker_daemon_observation_sha256", docker_daemon_observation_sha256), + ("binary_sha256", binary_sha256), + ): + if not _valid_digest(value): + raise TypeError(f"invalid {name}") + if ( + type(flint_commit_content_file_count) is not int + or flint_commit_content_file_count <= 0 + or type(flint_project_pinned_release_only_file_count) is not int + or flint_project_pinned_release_only_file_count <= 0 + ): + raise TypeError("FLINT source partition must be nonempty") + if oci_image_reference != OCI_IMAGE_REFERENCE_V1 or oci_platform != OCI_PLATFORM_V1: + raise TypeError("diagnostic build does not bind the pinned OCI manifest/platform") + if ( + type(rebuild_sha256s) is not tuple + or len(rebuild_sha256s) != 2 + or any(not _valid_digest(item) for item in rebuild_sha256s) + or rebuild_sha256s != (binary_sha256, binary_sha256) + ): + raise TypeError("invalid reproducible-build digests") + if type(host_trust) is not HostTrustBoundaryV1: + raise TypeError("invalid host trust boundary") + if pipeline_policy_identity != pipeline_policy_identity_v1(host_trust): + raise TypeError("pipeline policy is not the fixed diagnostic policy") + if ( + type(build_processes) is not tuple + or len(build_processes) != 2 + or any(type(item) is not DockerBuildExitedV1 for item in build_processes) + or any(item.returncode != 0 for item in build_processes) + ): + raise TypeError("invalid build process observations") + if ( + type(comparator) is not DiagnosticArbComparatorV1 + or comparator.structural_source_identity != structural_source_identity + or comparator.build_input_identity != build_input_identity + or comparator.pipeline_policy_identity != pipeline_policy_identity + or comparator.binary_sha256 != binary_sha256 + or comparator.rebuild_sha256s != rebuild_sha256s + ): + raise TypeError("comparator does not bind this diagnostic build") + if type(binary) is not bytes or hashlib.sha256(binary).digest() != binary_sha256: + raise TypeError("invalid owned binary") + for field_name, field_value in ( + ("structural_source_identity", structural_source_identity), + ("flint_commit_content_identity", flint_commit_content_identity), + ("flint_commit_content_file_count", flint_commit_content_file_count), + ( + "flint_project_pinned_release_only_identity", + flint_project_pinned_release_only_identity, + ), + ( + "flint_project_pinned_release_only_file_count", + flint_project_pinned_release_only_file_count, + ), + ("build_input_identity", build_input_identity), + ("formula_support_identity", formula_support_identity), + ("pipeline_policy_identity", pipeline_policy_identity), + ( + "docker_daemon_observation_sha256", + docker_daemon_observation_sha256, + ), + ("oci_image_reference", oci_image_reference), + ("oci_platform", oci_platform), + ("binary_sha256", binary_sha256), + ("rebuild_sha256s", rebuild_sha256s), + ("host_trust", host_trust), + ("build_processes", build_processes), + ("comparator", comparator), + ): + object.__setattr__(self, field_name, field_value) + object.__setattr__(self, "_binary", binary) + + @property + def binary(self) -> bytes: + return self._binary + + +@dataclass(frozen=True, init=False) +class DiagnosticPipelineObservationV1: + """Diagnostic BUILD plus diagnostic RUN; never a receipt or native proof.""" + + build_observation: DiagnosticBuildObservationV1 + invocation_identity: bytes + platform_identity: bytes + transcript: protocol.DecisionTranscriptV1 + run_claim: protocol.RunClaimV1 + _transcript_bytes: bytes + + def __init__( + self, + build_observation: DiagnosticBuildObservationV1, + invocation_identity: bytes, + platform_identity: bytes, + transcript: protocol.DecisionTranscriptV1, + run_claim: protocol.RunClaimV1, + transcript_bytes: bytes, + *, + _token: object, + ) -> None: + if _token is not _PIPELINE_OBSERVATION_TOKEN: + raise TypeError("DiagnosticPipelineObservationV1 is controller-only") + if type(build_observation) is not DiagnosticBuildObservationV1: + raise TypeError("invalid diagnostic build observation") + if not _valid_digest(invocation_identity) or not _valid_digest(platform_identity): + raise TypeError("invalid RUN observation identities") + if type(transcript) is not protocol.DecisionTranscriptV1: + raise TypeError("invalid transcript") + if type(run_claim) is not protocol.RunClaimV1: + raise TypeError("invalid run claim") + if ( + transcript.comparator_identity != build_observation.comparator.identity + or run_claim.job_identity != transcript.job_identity + or run_claim.comparator_identity != build_observation.comparator.identity + or run_claim.binary_identity != build_observation.binary_sha256 + or run_claim.invocation_identity != invocation_identity + or run_claim.platform_identity != platform_identity + or run_claim.transcript_identity != transcript.identity + ): + raise TypeError("run claim does not bind diagnostic observations") + if type(transcript_bytes) is not bytes or transcript.encode() != transcript_bytes: + raise TypeError("invalid owned transcript") + object.__setattr__(self, "build_observation", build_observation) + object.__setattr__(self, "invocation_identity", invocation_identity) + object.__setattr__(self, "platform_identity", platform_identity) + object.__setattr__(self, "transcript", transcript) + object.__setattr__(self, "run_claim", run_claim) + object.__setattr__(self, "_transcript_bytes", transcript_bytes) + + @property + def comparator(self) -> DiagnosticArbComparatorV1: + return self.build_observation.comparator + + @property + def structural_source_identity(self) -> bytes: + return self.build_observation.structural_source_identity + + @property + def flint_commit_content_identity(self) -> bytes: + return self.build_observation.flint_commit_content_identity + + @property + def flint_commit_content_file_count(self) -> int: + return self.build_observation.flint_commit_content_file_count + + @property + def flint_project_pinned_release_only_identity(self) -> bytes: + return self.build_observation.flint_project_pinned_release_only_identity + + @property + def flint_project_pinned_release_only_file_count(self) -> int: + return self.build_observation.flint_project_pinned_release_only_file_count + + @property + def build_input_identity(self) -> bytes: + return self.build_observation.build_input_identity + + @property + def formula_support_identity(self) -> bytes: + return self.build_observation.formula_support_identity + + @property + def pipeline_policy_identity(self) -> bytes: + return self.build_observation.pipeline_policy_identity + + @property + def docker_daemon_observation_sha256(self) -> bytes: + return self.build_observation.docker_daemon_observation_sha256 + + @property + def oci_image_reference(self) -> str: + return self.build_observation.oci_image_reference + + @property + def oci_platform(self) -> str: + return self.build_observation.oci_platform + + @property + def binary_sha256(self) -> bytes: + return self.build_observation.binary_sha256 + + @property + def rebuild_sha256s(self) -> tuple[bytes, bytes]: + return self.build_observation.rebuild_sha256s + + @property + def host_trust(self) -> HostTrustBoundaryV1: + return self.build_observation.host_trust + + @property + def build_processes(self) -> tuple[DockerBuildExitedV1, DockerBuildExitedV1]: + return self.build_observation.build_processes + + @property + def binary(self) -> bytes: + return self.build_observation.binary + + @property + def transcript_bytes(self) -> bytes: + return self._transcript_bytes + + +BuildResultV1: TypeAlias = ( + DiagnosticBuildObservationV1 + | PipelineBlockedV1 + | BuildRejectedV1 + | NonReproducibleBuildV1 +) + + +PipelineResultV1: TypeAlias = ( + DiagnosticPipelineObservationV1 + | PipelineBlockedV1 + | BuildRejectedV1 + | NonReproducibleBuildV1 + | ExecutionRejectedV1 + | TranscriptRejectedV1 +) + + +def invocation_identity_v1(request: executor.ExecutionRequestV1) -> bytes: + if type(request) is not executor.ExecutionRequestV1: + raise TypeError("request must be ExecutionRequestV1") + chunks: list[bytes] = [hashlib.sha256(request.executable).digest()] + chunks.append(len(request.argv).to_bytes(4, "big")) + chunks.extend(request.argv) + chunks.append(len(request.environment).to_bytes(4, "big")) + for key, value in request.environment: + chunks.extend((key, value)) + chunks.extend( + ( + request.cwd, + hashlib.sha256(request.stdin).digest(), + len(request.stdin).to_bytes(8, "big"), + request.umask.to_bytes(4, "big"), + ) + ) + for item in fields(request.limits): + chunks.append(getattr(request.limits, item.name).to_bytes(8, "big")) + return _identity(_INVOCATION_ID_LABEL_V1, tuple(chunks)) + + +def platform_identity_v1(report: executor.SupportedV1) -> bytes: + if type(report) is not executor.SupportedV1: + raise TypeError("report must be SupportedV1") + return _identity( + _PLATFORM_ID_LABEL_V1, + ( + report.platform.encode("ascii"), + report.sandbox_policy_release.encode("ascii"), + ), + ) + + +class _TreeMismatchV1(RuntimeError): + pass + + +def _write_all(descriptor: int, contents: bytes) -> None: + cursor = 0 + while cursor < len(contents): + written = os.write(descriptor, contents[cursor:]) + if written <= 0: + raise OSError("short write") + cursor += written + + +def _write_exact_file(root: Path, item: BuildSourceFileV1) -> None: + target = root / item.path + current = root + for part in PurePosixPath(item.path).parent.parts: + current = current / part + try: + current.mkdir(mode=0o755) + except FileExistsError: + metadata = current.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise _TreeMismatchV1("parent collision") + current.chmod(0o755) + descriptor = os.open( + target, + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + item.mode, + ) + try: + _write_all(descriptor, item.contents) + os.fchmod(descriptor, item.mode) + finally: + os.close(descriptor) + + +def _expected_directories(paths: set[str]) -> set[str]: + result = {"."} + for path in paths: + parent = PurePosixPath(path).parent + while str(parent) != ".": + result.add(str(parent)) + parent = parent.parent + return result + + +def _verify_exact_tree( + root: Path, + expected: dict[str, tuple[int, int, bytes]], +) -> None: + actual_files: set[str] = set() + actual_directories: set[str] = {"."} + for directory, directory_names, file_names in os.walk(root, followlinks=False): + base = Path(directory) + relative_base = base.relative_to(root) + metadata = base.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise _TreeMismatchV1("non-directory in tree") + if stat.S_IMODE(metadata.st_mode) != 0o755: + raise _TreeMismatchV1("directory mode drift") + for name in directory_names: + target = base / name + target_metadata = target.lstat() + if not stat.S_ISDIR(target_metadata.st_mode) or stat.S_ISLNK(target_metadata.st_mode): + raise _TreeMismatchV1("link or non-directory parent") + relative = (relative_base / name).as_posix() + actual_directories.add(relative) + for name in file_names: + target = base / name + relative = (relative_base / name).as_posix() + coordinate = expected.get(relative) + if coordinate is None: + raise _TreeMismatchV1("extra file") + metadata = target.lstat() + mode, length, digest = coordinate + if ( + not stat.S_ISREG(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_nlink != 1 + or stat.S_IMODE(metadata.st_mode) != mode + or metadata.st_size != length + ): + raise _TreeMismatchV1("file metadata drift") + hasher = hashlib.sha256() + with target.open("rb") as stream: + while chunk := stream.read(64 * 1024): + hasher.update(chunk) + if hasher.digest() != digest: + raise _TreeMismatchV1("file content drift") + actual_files.add(relative) + if actual_files != set(expected) or actual_directories != _expected_directories(set(expected)): + raise _TreeMismatchV1("tree shape drift") + + +def _read_build_output(directory: Path, maximum: int) -> bytes: + try: + names = tuple(item.name for item in directory.iterdir()) + except OSError as error: + raise _TreeMismatchV1("cannot list build output") from error + if names != (EVALUATOR_OUTPUT_NAME_V1,): + raise _TreeMismatchV1("build output must contain exactly one file") + directory_fd = os.open( + directory, + os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_CLOEXEC", 0), + ) + try: + descriptor = os.open( + EVALUATOR_OUTPUT_NAME_V1, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + dir_fd=directory_fd, + ) + except OSError as error: + os.close(directory_fd) + raise _TreeMismatchV1("cannot open exact build output") from error + try: + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_nlink != 1 + or stat.S_IMODE(before.st_mode) != 0o555 + or before.st_size <= 0 + or before.st_size > maximum + ): + raise _TreeMismatchV1("invalid build output metadata") + chunks: list[bytes] = [] + length = 0 + while True: + chunk = os.read(descriptor, min(64 * 1024, maximum + 1 - length)) + if not chunk: + break + chunks.append(chunk) + length += len(chunk) + if length > maximum: + raise _TreeMismatchV1("oversized build output") + after = os.fstat(descriptor) + coordinates_before = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + coordinates_after = ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) + if coordinates_before != coordinates_after or length != before.st_size: + raise _TreeMismatchV1("build output changed during observation") + return b"".join(chunks) + except OSError as error: + raise _TreeMismatchV1("cannot read exact build output") from error + finally: + os.close(descriptor) + os.close(directory_fd) + + +class ExecutionControllerV1(Protocol): + def probe(self) -> executor.CapabilityReportV1: ... + + def execute( + self, + request: executor.ExecutionRequestV1, + capability: executor.SupportedV1, + ) -> executor.ExecutionResultV1: ... + + +class ControlledPipelineV1: + def __init__( + self, + *, + build_backend: DockerBuildBackendV1, + execution_controller: ExecutionControllerV1 | None, + ) -> None: + self._build_backend = build_backend + self._execution_controller = execution_controller + + def build(self, request: PipelineRequestV1) -> BuildResultV1: + """Observe two fresh equal builds without requiring a RUN capability.""" + + if type(request) is not PipelineRequestV1: + raise PipelineInputErrorV1(PipelineInputReasonV1.WRONG_TYPE, "request") + try: + docker_report = self._build_backend.probe() + except Exception: + return PipelineBlockedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "Docker capability probe raised", + ) + if type(docker_report) is DockerUnsupportedV1: + return PipelineBlockedV1(docker_report.reason, docker_report.detail) + if type(docker_report) is not DockerSupportedV1: + return PipelineBlockedV1( + DockerBlockerReasonV1.BACKEND_CONTRACT, + "Docker capability report is not typed", + ) + + builds: list[tuple[bytes, DockerBuildExitedV1]] = [] + for attempt in (1, 2): + built = self._build_once(request, attempt) + if type(built) is BuildRejectedV1: + return built + builds.append(built) + first, second = builds + first_digest = hashlib.sha256(first[0]).digest() + second_digest = hashlib.sha256(second[0]).digest() + if first[0] != second[0]: + return NonReproducibleBuildV1(first_digest, second_digest) + + binary = first[0] + rebuild_sha256s = (first_digest, second_digest) + build_processes = (first[1], second[1]) + comparator = _derive_arb_comparator_for_build_v1( + request, + docker_report, + binary, + rebuild_sha256s, + build_processes, + ) + flint_partition = flint_source_content_partition_v1( + request.source_lock, + request.admitted_sources, + ) + return DiagnosticBuildObservationV1( + request.admitted_sources.identity, + flint_partition.commit_content_identity, + flint_partition.commit_content_file_count, + flint_partition.project_pinned_release_only_identity, + flint_partition.project_pinned_release_only_file_count, + request.build_sources.build_input_identity, + request.build_sources.formula_support_identity, + pipeline_policy_identity_v1(request.host_trust), + docker_report.daemon_observation_sha256, + docker_report.image_reference, + docker_report.platform, + first_digest, + rebuild_sha256s, + request.host_trust, + build_processes, + comparator, + binary, + _token=_BUILD_OBSERVATION_TOKEN, + ) + + def execute(self, request: PipelineRequestV1) -> PipelineResultV1: + if type(request) is not PipelineRequestV1: + raise PipelineInputErrorV1(PipelineInputReasonV1.WRONG_TYPE, "request") + build_observation = self.build(request) + if type(build_observation) is not DiagnosticBuildObservationV1: + return build_observation + if self._execution_controller is None: + return ExecutionRejectedV1( + ExecutionFailureReasonV1.BACKEND_CONTRACT, + "execution controller is unavailable", + ) + try: + execution_report = self._execution_controller.probe() + except Exception: + return ExecutionRejectedV1( + ExecutionFailureReasonV1.BACKEND_CONTRACT, + "executor capability probe raised", + ) + if type(execution_report) is executor.UnsupportedV1: + return ExecutionRejectedV1( + ExecutionFailureReasonV1.UNSUPPORTED, + execution_report, + ) + if type(execution_report) is not executor.SupportedV1: + return ExecutionRejectedV1( + ExecutionFailureReasonV1.BACKEND_CONTRACT, + execution_report, + ) + + # This is the exact first post-exit bytes object retained by BUILD. + binary = build_observation.binary + try: + invocation = executor.ExecutionRequestV1( + executable=binary, + argv=( + b"arb-evaluator", + b"--manifest-identity", + build_observation.comparator.identity.hex().encode("ascii"), + b"--job", + b"/dev/stdin", + ), + environment=((b"LC_ALL", b"C"), (b"TZ", b"UTC")), + cwd=b"/", + stdin=request.job.encode(), + umask=0o077, + limits=request.execution_limits, + ) + except executor.ExecutionRequestErrorV1 as error: + return ExecutionRejectedV1( + ExecutionFailureReasonV1.BACKEND_CONTRACT, + error, + ) + invocation_identity = invocation_identity_v1(invocation) + platform_identity = platform_identity_v1(execution_report) + try: + execution_result = self._execution_controller.execute( + invocation, + execution_report, + ) + except Exception: + return ExecutionRejectedV1( + ExecutionFailureReasonV1.BACKEND_CONTRACT, + "executor raised", + ) + if type(execution_result) is not executor.CompletedV1: + if not executor.result_matches_request_v1(execution_result, invocation): + return ExecutionRejectedV1( + ExecutionFailureReasonV1.BACKEND_CONTRACT, + execution_result, + ) + return ExecutionRejectedV1( + ExecutionFailureReasonV1.PROCESS_FAILED, + execution_result, + ) + if execution_result.binary_sha256 != build_observation.binary_sha256: + return ExecutionRejectedV1( + ExecutionFailureReasonV1.BINARY_MISMATCH, + execution_result, + ) + if not executor.result_matches_request_v1(execution_result, invocation): + return ExecutionRejectedV1( + ExecutionFailureReasonV1.BACKEND_CONTRACT, + execution_result, + ) + if execution_result.stderr: + return ExecutionRejectedV1( + ExecutionFailureReasonV1.STDERR_NOT_EMPTY, + execution_result, + ) + transcript_bytes = execution_result.stdout + try: + transcript = protocol.DecisionTranscriptV1.parse(transcript_bytes) + except protocol.ProtocolErrorV1 as error: + return TranscriptRejectedV1( + TranscriptFailureReasonV1.INVALID_WIRE, + str(error), + ) + if ( + transcript.encode() != transcript_bytes + or transcript.job_identity != request.job.identity + or transcript.domain_identity != request.job.domain.identity + or transcript.comparator_identity != build_observation.comparator.identity + or transcript.point_count != request.job.domain.point_count + ): + return TranscriptRejectedV1( + TranscriptFailureReasonV1.FOREIGN_BINDING, + "transcript does not bind the exact job/domain/comparator", + ) + try: + protocol.validate_witness_alignment_v1( + request.job.domain, + transcript.decision_bits, + transcript.point_count, + transcript.counters, + transcript.witness_store, + ) + except protocol.ProtocolErrorV1 as error: + return TranscriptRejectedV1( + TranscriptFailureReasonV1.FOREIGN_BINDING, + str(error), + ) + try: + run_claim = protocol.RunClaimV1.for_transcript( + request.job, + build_observation.comparator.manifest, + transcript, + build_observation.binary_sha256, + invocation_identity, + platform_identity, + ) + except protocol.ProtocolErrorV1 as error: + return TranscriptRejectedV1( + TranscriptFailureReasonV1.FOREIGN_BINDING, + str(error), + ) + return DiagnosticPipelineObservationV1( + build_observation, + invocation_identity, + platform_identity, + transcript, + run_claim, + transcript_bytes, + _token=_PIPELINE_OBSERVATION_TOKEN, + ) + + def _build_once( + self, + request: PipelineRequestV1, + attempt: int, + ) -> tuple[bytes, DockerBuildExitedV1] | BuildRejectedV1: + try: + with tempfile.TemporaryDirectory(prefix=f"labcolors-arb-build-v1-{attempt}-") as temporary: + root = Path(temporary).resolve() + inputs = root / "inputs" + workspace = root / "workspace" + build = root / "build" + output = root / "out" + for directory in (inputs, workspace, build, output): + directory.mkdir(mode=0o755) + directory.chmod(0o755) + + for lock, admitted in zip( + request.source_lock.sources, + request.admitted_sources.sources, + strict=True, + ): + destination = inputs / lock.root_prefix[:-1] + snapshot.materialize_source_archive( + lock, + admitted, + destination, + ) + destination.chmod(0o755) + workspace_files: list[BuildSourceFileV1] = [] + for item in request.build_sources.files: + if item.path == GENERATED_FORMULA_PATH_V1: + generated = BuildSourceFileV1( + "formula.generated.c", + item.mode, + item.contents, + ) + _write_exact_file(inputs, generated) + else: + _write_exact_file(workspace, item) + workspace_files.append(item) + build_request = DockerBuildRequestV1( + attempt, + root, + inputs, + workspace, + build, + output, + root / "container.cid", + _CONTAINER_NAME_PREFIX_V1 + + hashlib.sha256( + os.fsencode(root) + bytes((attempt,)) + ).hexdigest(), + ) + try: + process = self._build_backend.run_build(build_request) + except Exception: + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.BACKEND_CONTRACT, + ) + known_process_types = ( + DockerBuildExitedV1, + DockerBuildTimedOutV1, + DockerBuildOutputLimitV1, + DockerBuildObserverFailureV1, + DockerBuildCleanupFailureV1, + ) + if type(process) not in known_process_types: + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.BACKEND_CONTRACT, + ) + if type(process) is DockerBuildCleanupFailureV1: + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.CLEANUP_FAILED, + process, + ) + if type(process) is not DockerBuildExitedV1 or process.returncode != 0: + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.PROCESS_FAILED, + process, + ) + try: + for lock, admitted in zip( + request.source_lock.sources, + request.admitted_sources.sources, + strict=True, + ): + expected = { + item.path: (item.mode, item.length, item.sha256) + for item in admitted.files + } + _verify_exact_tree(inputs / lock.root_prefix[:-1], expected) + expected_workspace = { + item.path: ( + item.mode, + len(item.contents), + hashlib.sha256(item.contents).digest(), + ) + for item in workspace_files + } + _verify_exact_tree(workspace, expected_workspace) + generated = request.build_sources.generated_formula + _verify_exact_tree( + inputs, + { + "formula.generated.c": ( + 0o644, + len(generated), + hashlib.sha256(generated).digest(), + ), + **{ + f"{lock.root_prefix[:-1]}/{item.path}": ( + item.mode, + item.length, + item.sha256, + ) + for lock, admitted in zip( + request.source_lock.sources, + request.admitted_sources.sources, + strict=True, + ) + for item in admitted.files + }, + }, + ) + except _TreeMismatchV1: + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.INPUT_CHANGED, + process, + ) + try: + binary = _read_build_output( + output, + request.execution_limits.max_executable_bytes, + ) + executor.require_static_x86_64_elf_v1(binary) + except (OSError, _TreeMismatchV1, executor.ExecutionRequestErrorV1): + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.INVALID_OUTPUT, + process, + ) + return binary, process + except ( + OSError, + _TreeMismatchV1, + snapshot.SnapshotErrorV1, + BuildSourceAdmissionErrorV1, + ): + return BuildRejectedV1( + attempt, + BuildFailureReasonV1.BACKEND_CONTRACT, + ) diff --git a/proof/region/v1/arb/snapshot.py b/proof/region/v1/arb/snapshot.py new file mode 100644 index 00000000..a858e5e7 --- /dev/null +++ b/proof/region/v1/arb/snapshot.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Materialize admitted source bytes into a new normalized build snapshot.""" + +from __future__ import annotations + +import hashlib +import io +import os +import stat +import tarfile +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import NoReturn + +import provenance + + +# Archive timestamps are deliberately outside source admission. One epoch for +# every materialized node prevents Make-style freshness checks from observing +# extraction order; changing it is therefore a versioned snapshot-policy change. +SOURCE_SNAPSHOT_MTIME_NS_V1 = 0 + + +class SnapshotReasonV1(StrEnum): + FOREIGN_CAPABILITY = "foreign_capability" + INVALID_DESTINATION = "invalid_destination" + MATERIALIZATION_MISMATCH = "materialization_mismatch" + IO_FAILURE = "io_failure" + + +@dataclass(frozen=True) +class SnapshotErrorV1(RuntimeError): + reason: SnapshotReasonV1 + detail: str + + def __str__(self) -> str: + return f"{self.reason}: {self.detail}" + + +def _fail(reason: SnapshotReasonV1, detail: str) -> NoReturn: + raise SnapshotErrorV1(reason, detail) + + +@dataclass(frozen=True) +class MaterializedSourceTreeV1: + tree_identity: bytes + regular_file_count: int + regular_file_bytes: int + + +def _write_all(descriptor: int, payload: bytes) -> None: + offset = 0 + while offset < len(payload): + try: + written = os.write(descriptor, payload[offset:]) + except OSError: + _fail(SnapshotReasonV1.IO_FAILURE, "source write failed") + if written <= 0: + _fail(SnapshotReasonV1.IO_FAILURE, "short source write") + offset += written + + +def _ensure_parent(root: Path, relative_parent: Path) -> None: + current = root + for component in relative_parent.parts: + current = current / component + try: + os.mkdir(current, 0o755) + except FileExistsError: + try: + metadata = current.lstat() + except OSError: + _fail(SnapshotReasonV1.IO_FAILURE, "cannot inspect source directory") + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + _fail(SnapshotReasonV1.MATERIALIZATION_MISMATCH, "parent collision") + except OSError: + _fail(SnapshotReasonV1.IO_FAILURE, "cannot create source directory") + try: + os.chmod(current, 0o755, follow_symlinks=False) + except OSError: + _fail(SnapshotReasonV1.IO_FAILURE, "cannot normalize source directory") + + +def _set_exact_snapshot_time(path: Path, *, directory: bool) -> None: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + if directory: + flags |= getattr(os, "O_DIRECTORY", 0) + descriptor = -1 + try: + descriptor = os.open(path, flags) + before = os.fstat(descriptor) + expected_kind = stat.S_ISDIR if directory else stat.S_ISREG + if not expected_kind(before.st_mode): + _fail(SnapshotReasonV1.MATERIALIZATION_MISMATCH, "snapshot node kind") + os.utime( + descriptor, + ns=(SOURCE_SNAPSHOT_MTIME_NS_V1, SOURCE_SNAPSHOT_MTIME_NS_V1), + ) + after = os.fstat(descriptor) + if ( + (after.st_dev, after.st_ino) != (before.st_dev, before.st_ino) + or not expected_kind(after.st_mode) + or after.st_mtime_ns != SOURCE_SNAPSHOT_MTIME_NS_V1 + ): + _fail(SnapshotReasonV1.IO_FAILURE, "source timestamp postcondition") + except SnapshotErrorV1: + raise + except OSError: + _fail(SnapshotReasonV1.IO_FAILURE, "cannot normalize source timestamps") + finally: + if descriptor >= 0: + os.close(descriptor) + + +def _normalize_snapshot_times(root: Path, relative_paths: set[str]) -> None: + directories = {root} + for relative in sorted(relative_paths): + target = root / relative + _set_exact_snapshot_time(target, directory=False) + parent = target.parent + while parent != root: + directories.add(parent) + parent = parent.parent + for directory in sorted( + directories, + key=lambda item: (-len(item.relative_to(root).parts), item.as_posix()), + ): + _set_exact_snapshot_time(directory, directory=True) + + +def materialize_source_archive( + expected: provenance.SourceReleaseLockV1, + admitted: provenance.SafeSourceArchiveV1, + destination: Path, +) -> MaterializedSourceTreeV1: + """Write only regular files from the exact bytes owned by `admitted`.""" + + if type(expected) is not provenance.SourceReleaseLockV1: + raise TypeError("expected must be SourceReleaseLockV1") + if type(admitted) is not provenance.SafeSourceArchiveV1: + raise TypeError("admitted must be SafeSourceArchiveV1") + if not isinstance(destination, Path): + raise TypeError("destination must be Path") + if admitted.source_lock_identity != expected.identity: + _fail(SnapshotReasonV1.FOREIGN_CAPABILITY, "source lock identity") + + root_name = expected.root_prefix[:-1] + if destination.name != root_name or destination.exists() or destination.is_symlink(): + _fail(SnapshotReasonV1.INVALID_DESTINATION, "destination must be a new release root") + try: + parent = destination.parent.resolve(strict=True) + except (OSError, RuntimeError): + _fail(SnapshotReasonV1.INVALID_DESTINATION, "destination parent unavailable") + if not parent.is_dir(): + _fail(SnapshotReasonV1.INVALID_DESTINATION, "destination parent is not a directory") + destination = parent / destination.name + + replayed, raw_tar = provenance.replay_admitted_source_archive_v1( + expected, + admitted, + ) + if ( + replayed.archive_sha256 != admitted.archive_sha256 + or replayed.tree_identity != admitted.tree_identity + or replayed.regular_file_count != admitted.regular_file_count + or replayed.regular_file_bytes != admitted.regular_file_bytes + or replayed.files != admitted.files + ): + _fail(SnapshotReasonV1.FOREIGN_CAPABILITY, "archive replay drift") + expected_files = {item.path: item for item in admitted.files} + seen: set[str] = set() + try: + os.mkdir(destination, 0o755) + with tarfile.open(fileobj=io.BytesIO(raw_tar), mode="r:") as archive: + for member in archive: + if not member.isreg(): + continue + relative = member.name[len(expected.root_prefix) :] + coordinate = expected_files.get(relative) + if coordinate is None or relative in seen: + _fail(SnapshotReasonV1.MATERIALIZATION_MISMATCH, relative) + stream = archive.extractfile(member) + if stream is None: + _fail(SnapshotReasonV1.MATERIALIZATION_MISMATCH, relative) + target = destination / relative + _ensure_parent(destination, Path(relative).parent) + flags = ( + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + descriptor = os.open(target, flags, coordinate.mode) + try: + hasher = hashlib.sha256() + length = 0 + while True: + chunk = stream.read(provenance.READ_CHUNK_BYTES) + if not chunk: + break + length += len(chunk) + if length > coordinate.length: + _fail(SnapshotReasonV1.MATERIALIZATION_MISMATCH, relative) + hasher.update(chunk) + _write_all(descriptor, chunk) + if length != coordinate.length or hasher.digest() != coordinate.sha256: + _fail(SnapshotReasonV1.MATERIALIZATION_MISMATCH, relative) + os.fchmod(descriptor, coordinate.mode) + finally: + os.close(descriptor) + seen.add(relative) + except SnapshotErrorV1: + raise + except (OSError, tarfile.TarError): + _fail(SnapshotReasonV1.IO_FAILURE, "materialization failed") + if seen != set(expected_files): + _fail(SnapshotReasonV1.MATERIALIZATION_MISMATCH, "missing source file") + _normalize_snapshot_times(destination, seen) + return MaterializedSourceTreeV1( + admitted.tree_identity, + admitted.regular_file_count, + admitted.regular_file_bytes, + ) diff --git a/proof/region/v1/arb/tests/gate.py b/proof/region/v1/arb/tests/gate.py new file mode 100644 index 00000000..00e45f70 --- /dev/null +++ b/proof/region/v1/arb/tests/gate.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Run the complete fast Arb contract with an exact skip manifest.""" + +from __future__ import annotations + +import hashlib +import sys +import unittest +from collections.abc import Iterator +from pathlib import Path + + +TEST_DIRECTORY = Path(__file__).resolve().parent +REPO = Path(__file__).resolve().parents[5] +sys.path.insert(0, str(REPO)) +EXPECTED_TEST_INVENTORY_SHA256 = ( + "7cccde0a6088de17be742af5207cc16229cf40d1960f371482d2a20d89995a80" +) +_EVALUATOR_REASON = "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary" +EXPECTED_SKIPS = frozenset( + { + ( + f"test_evaluator_source.ExactBoundaryRuntimeTests.{name}", + _EVALUATOR_REASON, + ) + for name in ( + "test_black_exact_zero_runs_through_job_parser_formula_and_closed_driver", + "test_cli_requires_one_nonzero_lowercase_manifest_identity", + "test_frozen_seam_cube_resolves_one_inside_and_511_outside", + "test_global_pregrant_is_never_transferred_between_points", + "test_multisegment_exact_trace_selects_first_canonical_branch", + "test_resource_witness_accounts_for_work_consumed_on_earlier_rungs", + "test_spd_admission_is_exact_across_the_full_binary64_exponent_range", + "test_subminimum_precision_is_unresolved_and_a_later_valid_rung_recovers", + "test_zero_grant_emits_canonical_resource_witnesses", + ) + } + | { + ( + "test_executor.NativeLinuxIntegrationTests." + "test_real_kernel_success_output_timeout_signal_oom_and_cleanup", + "requires Linux and an explicit delegated cgroup v2 parent", + ), + ( + "test_pipeline.NativeBuildIntegrationTests." + "test_real_two_builds_and_ephemeral_evaluator_runtime_tests", + "requires Linux, Docker, the native binary path, and all three exact source archives", + ), + ( + "test_pipeline.NativePipelineIntegrationTests." + "test_prepared_two_build_binary_runs_through_controlled_pipeline", + "requires Linux, the native binary path, and an explicit " + "delegated cgroup v2 parent", + ), + } +) + + +def _iter_tests_v1(suite: unittest.TestSuite) -> Iterator[unittest.TestCase]: + for item in suite: + if isinstance(item, unittest.TestSuite): + yield from _iter_tests_v1(item) + elif isinstance(item, unittest.TestCase): + yield item + else: + raise TypeError("suite contains a non-test object") + + +def _inventory_preimage_v1(test_ids: tuple[str, ...]) -> bytes: + return b"".join(test_id.encode("utf-8") + b"\n" for test_id in sorted(test_ids)) + + +def test_inventory_sha256_v1(suite: unittest.TestSuite) -> str: + test_ids = tuple(test.id() for test in _iter_tests_v1(suite)) + return hashlib.sha256(_inventory_preimage_v1(test_ids)).hexdigest() + + +def run_exact_suite_v1( + suite: unittest.TestSuite, + *, + expected_inventory_sha256: str, + expected_skips: frozenset[tuple[str, str]], + verbosity: int = 2, +) -> int: + tests = tuple(_iter_tests_v1(suite)) + test_ids = tuple(test.id() for test in tests) + actual_inventory_sha256 = hashlib.sha256( + _inventory_preimage_v1(test_ids) + ).hexdigest() + if ( + not tests + or len(set(test_ids)) != len(test_ids) + or actual_inventory_sha256 != expected_inventory_sha256 + ): + print( + "Arb test inventory drift: " + f"count={len(tests)} sha256={actual_inventory_sha256} " + f"expected={expected_inventory_sha256}", + file=sys.stderr, + ) + return 1 + result = unittest.TextTestRunner(verbosity=verbosity).run(suite) + actual_skips = frozenset((test.id(), reason) for test, reason in result.skipped) + if actual_skips != expected_skips: + print(f"unexpected skips: {sorted(actual_skips - expected_skips)!r}", file=sys.stderr) + print(f"missing skips: {sorted(expected_skips - actual_skips)!r}", file=sys.stderr) + return 1 + if ( + result.failures + or result.errors + or result.expectedFailures + or result.unexpectedSuccesses + or not result.wasSuccessful() + ): + print( + "proof suite contains failures, errors, expected failures, or " + "unexpected successes", + file=sys.stderr, + ) + return 1 + print( + f"Arb fast gate: {len(tests)} tests, " + f"inventory {actual_inventory_sha256}, " + f"exact {len(actual_skips)}-skip manifest" + ) + return 0 + + +def main() -> int: + suite = unittest.defaultTestLoader.discover( + str(TEST_DIRECTORY), + pattern="test_*.py", + ) + return run_exact_suite_v1( + suite, + expected_inventory_sha256=EXPECTED_TEST_INVENTORY_SHA256, + expected_skips=EXPECTED_SKIPS, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/proof/region/v1/arb/tests/native_gate.py b/proof/region/v1/arb/tests/native_gate.py new file mode 100644 index 00000000..1d90e430 --- /dev/null +++ b/proof/region/v1/arb/tests/native_gate.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Require one exact native integration lane without skips.""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + + +REPO = Path(__file__).resolve().parents[5] +sys.path.insert(0, str(REPO)) + +from proof.region.v1.arb.tests import gate # noqa: E402 +from proof.region.v1.arb.tests.test_executor import ( # noqa: E402 + NativeLinuxIntegrationTests, +) +from proof.region.v1.arb.tests.test_pipeline import ( # noqa: E402 + NativeBuildIntegrationTests, + NativePipelineIntegrationTests, +) + + +_MODES = { + "build": ( + (NativeBuildIntegrationTests,), + "a6f8057d55a19bee9e924fa3bea2f082455ece0b8a9be5caf022b4a61aa9d15e", + ), + "executor": ( + (NativeLinuxIntegrationTests, NativePipelineIntegrationTests), + "0a7135fc2c259f125aa3cb692ea480550549d3aed5fdb95c47a3ddc999969a4d", + ), +} + + +def main() -> int: + if len(sys.argv) != 2 or sys.argv[1] not in _MODES: + print("usage: native_gate.py {build|executor}", file=sys.stderr) + return 64 + test_cases, inventory = _MODES[sys.argv[1]] + suite = unittest.TestSuite( + unittest.defaultTestLoader.loadTestsFromTestCase(test_case) + for test_case in test_cases + ) + return gate.run_exact_suite_v1( + suite, + expected_inventory_sha256=inventory, + expected_skips=frozenset(), + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/proof/region/v1/arb/tests/runtime_gate.py b/proof/region/v1/arb/tests/runtime_gate.py new file mode 100644 index 00000000..eab5bd1b --- /dev/null +++ b/proof/region/v1/arb/tests/runtime_gate.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Require the exact evaluator runtime suite with no vacuous outcomes.""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + + +REPO = Path(__file__).resolve().parents[5] +sys.path.insert(0, str(REPO)) + +from proof.region.v1.arb.tests import gate # noqa: E402 +from proof.region.v1.arb.tests.test_evaluator_source import ( # noqa: E402 + ExactBoundaryRuntimeTests, +) + + +EXPECTED_RUNTIME_INVENTORY_SHA256 = ( + "bc169a72a472a67e206250f755006085fa204646ee76fe7c6e8752db072aa73a" +) + + +def main() -> int: + suite = unittest.defaultTestLoader.loadTestsFromTestCase( + ExactBoundaryRuntimeTests + ) + return gate.run_exact_suite_v1( + suite, + expected_inventory_sha256=EXPECTED_RUNTIME_INVENTORY_SHA256, + expected_skips=frozenset(), + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/proof/region/v1/arb/tests/test_build_recipe.py b/proof/region/v1/arb/tests/test_build_recipe.py new file mode 100644 index 00000000..c66005e9 --- /dev/null +++ b/proof/region/v1/arb/tests/test_build_recipe.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Anti-vacuum contract for the offline Arb dependency build.""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +import unittest +from pathlib import Path + +from proof.region.v1.arb.tests import gate as arb_gate + + +ARB = Path(__file__).resolve().parents[1] +BUILD = ARB / "build.sh" +WORKFLOW = ARB.parents[3] / ".github" / "workflows" / "arb.yml" +RECIPE_REJECTION_TIMEOUT_SECONDS = 5 + + +class ArbBuildRecipeTests(unittest.TestCase): + def test_pr_gate_requires_a_disposable_exact_workflow_runner(self) -> None: + source = WORKFLOW.read_text(encoding="utf-8") + runner_contracts = [ + line.strip() + for line in source.splitlines() + if line.lstrip().startswith("runs-on:") + ] + + self.assertEqual( + runner_contracts, + ["runs-on: [self-hosted, Linux, X64, labcolors-ephemeral]"], + ) + self.assertIn("proof/region/v1/arb/tests/gate.py", source) + self.assertIn("proof/region/v1/arb/tests/native_gate.py", source) + self.assertEqual(source.count("- .github/workflows/arb.yml"), 2) + self.assertNotIn("arb-proof-observation.yml", source) + for required in ( + 'original_userns="$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns)"', + 'echo "LABCOLORS_APPARMOR_USERNS_V1=$original_userns"', + "sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0", + 'kernel.apparmor_restrict_unprivileged_userns=$LABCOLORS_APPARMOR_USERNS_V1', + 'mkdir "$scope/tasks" "$scope/proof"', + "printf '+memory +pids' > \"$scope/cgroup.subtree_control\"", + "printf '+memory +pids' > \"$scope/proof/cgroup.subtree_control\"", + "printf '2' > \"$scope/proof/pids.max\"", + 'mkdir "$scope/proof/observer"', + '"$scope/proof/cgroup.subtree_control"', + 'scope="/sys/fs/cgroup/labcolors-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"', + 'binary="$RUNNER_TEMP/arb-native-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"', + 'echo "LABCOLORS_CGROUP_SCOPE_V1=$scope"', + 'echo "LABCOLORS_EXECUTOR_CGROUP_V1=$scope/proof"', + 'echo "LABCOLORS_ARB_NATIVE_BINARY=$binary"', + '"$LABCOLORS_CGROUP_SCOPE_V1/tasks/cgroup.procs"', + '"$LABCOLORS_EXECUTOR_CGROUP_V1/observer/cgroup.procs"', + "native_gate.py build", + "native_gate.py executor", + "exec python3", + 'stat --format=%a "$LABCOLORS_ARB_NATIVE_BINARY"', + 'rm -f -- "$LABCOLORS_ARB_NATIVE_BINARY"', + '"$LABCOLORS_CGROUP_SCOPE_V1/cgroup.kill"', + "'populated 0'", + "for child in proof/observer proof tasks", + 'sudo rmdir "$LABCOLORS_CGROUP_SCOPE_V1/$child"', + 'sudo rmdir "$LABCOLORS_CGROUP_SCOPE_V1"', + ): + with self.subTest(required=required): + self.assertIn(required, source) + self.assertNotIn("grep --ignore-case --quiet skipped", source) + self.assertNotIn("python3 -m unittest", source) + self.assertLess( + source.index("proof/region/v1/arb/tests/gate.py"), + source.index("LABCOLORS_EXECUTOR_CGROUP_V1=$scope/proof"), + ) + + def test_pr_gate_cannot_green_skip_a_fork_without_execution(self) -> None: + source = WORKFLOW.read_text(encoding="utf-8") + + self.assertNotIn("github.event.pull_request.head.repo.full_name", source) + + def test_exact_suite_gate_rejects_expected_failure(self) -> None: + class BrokenRequiredTest(unittest.TestCase): + @unittest.expectedFailure + def test_required(self) -> None: + self.fail("broken") + + suite = unittest.defaultTestLoader.loadTestsFromTestCase(BrokenRequiredTest) + expected = arb_gate.test_inventory_sha256_v1(suite) + + self.assertEqual( + arb_gate.run_exact_suite_v1( + suite, + expected_inventory_sha256=expected, + expected_skips=frozenset(), + verbosity=0, + ), + 1, + ) + + def test_exact_suite_gate_rejects_empty_or_same_count_replacement(self) -> None: + class RequiredTest(unittest.TestCase): + def test_required(self) -> None: + pass + + class ReplacementTest(unittest.TestCase): + def test_replacement(self) -> None: + pass + + empty = unittest.TestSuite() + self.assertEqual( + arb_gate.run_exact_suite_v1( + empty, + expected_inventory_sha256=hashlib.sha256(b"").hexdigest(), + expected_skips=frozenset(), + verbosity=0, + ), + 1, + ) + required = unittest.defaultTestLoader.loadTestsFromTestCase(RequiredTest) + replacement = unittest.defaultTestLoader.loadTestsFromTestCase(ReplacementTest) + self.assertEqual(required.countTestCases(), replacement.countTestCases()) + self.assertEqual( + arb_gate.run_exact_suite_v1( + replacement, + expected_inventory_sha256=arb_gate.test_inventory_sha256_v1(required), + expected_skips=frozenset(), + verbosity=0, + ), + 1, + ) + + def test_recipe_is_offline_static_and_platform_explicit(self) -> None: + source = BUILD.read_text(encoding="utf-8") + + for required in ( + "/usr/bin/env -i", + "LC_BUILD_ENV_V1=1", + 'require_directory "$inputs/gmp-6.3.0"', + 'require_directory "$inputs/mpfr-4.2.2"', + 'require_directory "$inputs/flint-3.6.0"', + 'require_regular "$workspace/proof/region/v1/arb/evaluator/formula.h"', + "9958f20c8ca598625db0593a45f8f8bc79e4b2f22b53263b6c32d78a5e1d2693", + "-I.", + "--build=x86_64-pc-linux-gnu", + "--host=x86_64-pc-linux-gnu", + "--disable-shared", + "--enable-static", + "--disable-assembly", + "--enable-formally-proven-code", + "--disable-lto", + "--enable-assert", + "-fno-fast-math", + "-ffp-contract=off", + "-fno-lto", + "-std=gnu17", + "-march=x86-64", + "-mtune=generic", + "-Wl,--build-id=none", + "make check", + "readelf", + ): + with self.subTest(required=required): + self.assertIn(required, source) + + for forbidden in ( + "curl ", + "wget ", + "git clone", + "apt-get", + "brew ", + "tar --extract", + "-ffast-math", + "-march=native", + "-flto", + ): + with self.subTest(forbidden=forbidden): + self.assertNotIn(forbidden, source) + + self.assertIn( + 'if ! /usr/bin/readelf -l "$build/arb-evaluator-v1" ' + '> "$build/program-headers"; then', + source, + ) + self.assertIn( + 'if ! /usr/bin/readelf -d "$build/arb-evaluator-v1" ' + '> "$build/dynamic-section"; then', + source, + ) + self.assertNotIn("readelf -l \"$build/arb-evaluator-v1\" |", source) + self.assertNotIn("readelf -d \"$build/arb-evaluator-v1\" 2>&1 |", source) + + def test_recipe_rejects_ambient_or_incomplete_invocation_before_build(self) -> None: + self.assertTrue(os.access(BUILD, os.X_OK), BUILD) + result = subprocess.run( + [str(BUILD)], + check=False, + capture_output=True, + env={ + "PATH": os.environ.get("PATH", ""), + "UNDECLARED": "must-not-be-observed", + }, + stdin=subprocess.DEVNULL, + timeout=RECIPE_REJECTION_TIMEOUT_SECONDS, + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, b"") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/proof/region/v1/arb/tests/test_evaluator_source.py b/proof/region/v1/arb/tests/test_evaluator_source.py new file mode 100644 index 00000000..15a705de --- /dev/null +++ b/proof/region/v1/arb/tests/test_evaluator_source.py @@ -0,0 +1,807 @@ +#!/usr/bin/env python3 +"""Hostile source contract for the standalone Arb evaluator.""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ARB = Path(__file__).resolve().parents[1] +EVALUATOR = ARB / "evaluator" +REPO = ARB.parents[3] +FORMULA = REPO / "crates/labcolors-core/contracts/contextual-region-formula-v1.lcir" +GENERATOR = EVALUATOR / "formula.py" +# CI watchdogs bound broken test processes; they are not performance claims. +# Change them only with a measured exact native-gate workload and its job budget. +GENERATOR_TIMEOUT_SECONDS = 60 +EVALUATOR_TIMEOUT_SECONDS = 300 +sys.path.insert(0, str(REPO / "proof/region/v1")) + +from region_proof_protocol import ( # noqa: E402 + BoundaryUnprovenWitnessV1, + ComparatorBudgetV1, + ComparatorKindV1, + ComparatorManifestV2, + ContextualRegionDefinitionV1, + DecisionTranscriptV1, + DecisionV1, + ExactZeroSignalTraceV1, + ProofJobV1, + ProofPolicyV1, + ReducedDomainManifestV1, + ResourceLimitWitnessV1, +) + + +def generate(source: bytes) -> subprocess.CompletedProcess[bytes]: + with tempfile.TemporaryDirectory() as temporary: + formula = Path(temporary) / "formula.lcir" + formula.write_bytes(source) + return subprocess.run( + [sys.executable, str(GENERATOR), str(formula)], + check=False, + capture_output=True, + stdin=subprocess.DEVNULL, + timeout=GENERATOR_TIMEOUT_SECONDS, + env={ + "PATH": os.environ.get("PATH", ""), + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHASHSEED": "0", + }, + ) + + +def run_evaluator( + command: list[str] | tuple[str, ...], + stdin: bytes, +) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + command, + input=stdin, + check=False, + capture_output=True, + timeout=EVALUATOR_TIMEOUT_SECONDS, + ) + + +def assert_transcript_wire_coordinates( + case: unittest.TestCase, + wire: bytes, + transcript: DecisionTranscriptV1, + manifest_identity: bytes, +) -> None: + decision_bits = transcript.decision_bits + accounting_digest = transcript.accounting_digest + case.assertEqual(wire[:8], b"LCTRN1\0\0") + case.assertEqual(wire[72:104], manifest_identity) + accounting_offset = 160 + len(decision_bits) + case.assertEqual( + wire[accounting_offset : accounting_offset + 32], + accounting_digest, + ) + + +class FormulaGeneratorTests(unittest.TestCase): + def test_registered_formula_generates_one_deterministic_c_program(self) -> None: + source = FORMULA.read_bytes() + first = generate(source) + second = generate(source) + + self.assertEqual(first.returncode, 0, first.stderr.decode()) + self.assertEqual(second.returncode, 0, second.stderr.decode()) + self.assertEqual(first.stdout, second.stdout) + self.assertEqual( + hashlib.sha256(first.stdout).hexdigest(), + "9958f20c8ca598625db0593a45f8f8bc79e4b2f22b53263b6c32d78a5e1d2693", + ) + self.assertIn(b"lc_formula_point", first.stdout) + self.assertIn(b"lc_formula_segment", first.stdout) + self.assertIn(b"lc_formula_singleton", first.stdout) + self.assertNotIn(b"double", first.stdout) + + def test_generator_rejects_canonical_semantic_and_driver_mutations(self) -> None: + source = FORMULA.read_bytes() + mutations = ( + (b"labcolors_exact_real_ssa 1", b"labcolors_exact_real_ssa 2"), + (b"operator add 2 real exact_x_plus_y", b"operator add 2 real exact_x_minus_y"), + (b"node xyz_x_r real mul srgb_m00 linear_r", b"node xyz_x_r real add srgb_m00 linear_r"), + (b"literal p1_7 3ffb333333333333", b"literal p1_7 3ffb333333333334"), + (b"rule boundary inclusive", b"rule boundary exclusive"), + (b"point_nodes 226", b"point_nodes 225"), + ) + for needle, replacement in mutations: + with self.subTest(replacement=replacement): + self.assertIn(needle, source) + result = generate(source.replace(needle, replacement, 1)) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, b"") + + for mutant in (source + b"\n", source.replace(b"\n", b"\r\n", 1)): + result = generate(mutant) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, b"") + + def test_generator_is_independent_from_python_protocol_and_controller(self) -> None: + source = GENERATOR.read_text(encoding="utf-8") + for forbidden in ( + "region_proof_protocol", + "controller", + "import numpy", + "import scipy", + ): + self.assertNotIn(forbidden, source) + + +class StandaloneSourceTests(unittest.TestCase): + def test_evaluator_has_an_independent_wire_hash_interval_and_region_path(self) -> None: + required = ( + "main.c", + "wire.c", + "wire.h", + "hash.c", + "hash.h", + "interval.c", + "interval.h", + "region.c", + "region.h", + ) + for name in required: + with self.subTest(name=name): + self.assertTrue((EVALUATOR / name).is_file(), name) + + joined = "\n".join( + (EVALUATOR / name).read_text(encoding="utf-8") + for name in required + ) + for forbidden in ( + "region_proof_protocol", + "controller.py", + "arb_set_d(", + "strtod(", + "#include ", + " pow(", + " sqrt(", + "epsilon", + "midpoint", + "fallback", + ): + self.assertNotIn(forbidden, joined) + self.assertIn("arb_set_fmpz_2exp", joined) + self.assertIn("arb_get_interval_fmpz_2exp", joined) + self.assertIn("LCTRN1", joined) + self.assertNotIn("LCARO1", joined) + self.assertIn("--manifest-identity", joined) + + def test_closed_boundary_and_typed_unresolved_states_are_structural(self) -> None: + region = (EVALUATOR / "region.c").read_text(encoding="utf-8") + header = (EVALUATOR / "region.h").read_text(encoding="utf-8") + + for outcome in ( + "LC_REGION_INSIDE", + "LC_REGION_OUTSIDE", + "LC_REGION_BOUNDARY_UNPROVEN", + "LC_REGION_RESOURCE_LIMIT_REACHED", + ): + self.assertIn(outcome, header) + self.assertIn("arb_is_nonpositive", region) + self.assertIn("arb_is_positive", region) + self.assertIn("arb_intersection", region) + self.assertNotIn("arb_contains_zero(f)", region) + + def test_subminimum_flint_precision_never_enters_the_formula(self) -> None: + region = (EVALUATOR / "region.c").read_text(encoding="utf-8") + evaluator = region[region.index("lc_region_evaluate_rgb(") :] + + guard = evaluator.index("if (precision < 2)") + formula_call = evaluator.index("lc_formula_point(") + self.assertLess(guard, formula_call) + self.assertIn("minimum working precision", evaluator[:formula_call]) + + decision = region[ + region.index("lc_region_decide(") : region.index("lc_region_evaluate_rgb(") + ] + decision_guard = decision.index("if (precision < 2)") + singleton_dispatch = decision.index("if (region->knot_count == 1)") + self.assertLess(decision_guard, singleton_dispatch) + + def test_sha256_has_literal_standard_vectors_and_no_external_crypto(self) -> None: + source = (EVALUATOR / "hash.c").read_text(encoding="utf-8") + header = (EVALUATOR / "hash.h").read_text(encoding="utf-8") + self.assertIn("lc_sha256", header) + self.assertIn("0x6a09e667", source) + self.assertNotIn("openssl", source.lower()) + + +class ExactBoundaryRuntimeTests(unittest.TestCase): + @unittest.skipUnless( + os.environ.get("LABCOLORS_ARB_EVALUATOR"), + "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary", + ) + def test_cli_requires_one_nonzero_lowercase_manifest_identity(self) -> None: + executable = os.environ["LABCOLORS_ARB_EVALUATOR"] + valid = "ab" + "00" * 31 + invalid_invocations = ( + (), + ("--manifest-identity", "0" * 64, "--job", "/dev/stdin"), + ("--manifest-identity", valid.upper(), "--job", "/dev/stdin"), + ("--manifest-identity", "g" + valid[1:], "--job", "/dev/stdin"), + ("--manifest-identity", valid[:-1], "--job", "/dev/stdin"), + ("--manifest-identity", valid + "0", "--job", "/dev/stdin"), + ("--manifest-identity", valid, "--job", "job.bin"), + ("--manifest", valid, "--job", "/dev/stdin"), + ("--manifest-identity", valid, "--job", "/dev/stdin", "extra"), + ) + for arguments in invalid_invocations: + with self.subTest(arguments=arguments): + result = run_evaluator((executable, *arguments), b"") + self.assertEqual(result.returncode, 64) + self.assertEqual(result.stdout, b"") + + accepted = run_evaluator( + ( + executable, + "--manifest-identity", + valid, + "--job", + "/dev/stdin", + ), + b"", + ) + self.assertEqual(accepted.returncode, 1) + self.assertEqual(accepted.stdout, b"") + self.assertIn(b"job read failed", accepted.stderr) + + @unittest.skipUnless( + os.environ.get("LABCOLORS_ARB_EVALUATOR"), + "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary", + ) + def test_black_exact_zero_runs_through_job_parser_formula_and_closed_driver(self) -> None: + registered = ContextualRegionDefinitionV1.parse( + (REPO / "proof/region/v1/fixtures/v5b2b-definition-0a8d1c3d.bin").read_bytes() + ) + zero = bytes(8) + fields = registered.fields[:21] + ((1).to_bytes(8, "big"),) + (zero,) * 4 + definition = ContextualRegionDefinitionV1(fields, 1) + policy = ProofPolicyV1( + 1, + ( + ComparatorBudgetV1(ComparatorKindV1.ARB, (128,), 1, 1), + ComparatorBudgetV1(ComparatorKindV1.MPFI, (192,), 1, 1), + ), + ) + job = ProofJobV1( + definition, + FORMULA.read_bytes(), + ReducedDomainManifestV1.from_ordinals((0,)), + policy, + ) + manifest = ComparatorManifestV2( + ComparatorKindV1.ARB, + *(hashlib.sha256(f"arb-manifest-{index}".encode()).digest() for index in range(10)), + ) + executable = os.environ["LABCOLORS_ARB_EVALUATOR"] + result = run_evaluator( + [ + executable, + "--manifest-identity", + manifest.identity.hex(), + "--job", + "/dev/stdin", + ], + job.encode(), + ) + + self.assertEqual(result.returncode, 0, result.stderr.decode()) + self.assertEqual(result.stderr, b"") + transcript = DecisionTranscriptV1.parse(result.stdout) + self.assertEqual(transcript.encode(), result.stdout) + assert_transcript_wire_coordinates( + self, + result.stdout, + transcript, + manifest.identity, + ) + self.assertEqual(transcript.job_identity, job.identity) + self.assertEqual(transcript.domain_identity, job.domain.identity) + self.assertEqual(transcript.comparator_identity, manifest.identity) + self.assertEqual(tuple(transcript.iter_decisions()), (DecisionV1.INSIDE,)) + self.assertEqual(transcript.counters, (1, 0, 0, 0)) + self.assertEqual(transcript.exact_equality_count, 1) + witnesses = tuple(transcript.iter_witnesses()) + self.assertEqual(len(witnesses), 1) + self.assertIs(type(witnesses[0]), ExactZeroSignalTraceV1) + self.assertEqual(witnesses[0].ordinal, 0) + self.assertEqual( + witnesses[0].trace_digest, + hashlib.sha256( + b"labcolors.proof-region.exact-zero-signal-trace.v1\0" + + job.identity + + (0).to_bytes(4, "big") + + (0).to_bytes(8, "big") + ).digest(), + ) + + alternate_manifest = ComparatorManifestV2( + ComparatorKindV1.ARB, + *(hashlib.sha256(f"arb-alternate-{index}".encode()).digest() for index in range(10)), + ) + alternate = run_evaluator( + [ + executable, + "--manifest-identity", + alternate_manifest.identity.hex(), + "--job", + "/dev/stdin", + ], + job.encode(), + ) + self.assertEqual(alternate.returncode, 0, alternate.stderr.decode()) + alternate_transcript = DecisionTranscriptV1.parse(alternate.stdout) + self.assertEqual( + tuple(alternate_transcript.iter_decisions()), + tuple(transcript.iter_decisions()), + ) + self.assertEqual(alternate_transcript.counters, transcript.counters) + self.assertEqual(tuple(alternate_transcript.iter_witnesses()), witnesses) + self.assertEqual(alternate_transcript.comparator_identity, alternate_manifest.identity) + self.assertNotEqual(alternate_transcript.accounting_digest, transcript.accounting_digest) + self.assertNotEqual(alternate.stdout, result.stdout) + + corrupted = bytearray(job.encode()) + corrupted[-1] ^= 1 + rejected = run_evaluator( + [ + executable, + "--manifest-identity", + manifest.identity.hex(), + "--job", + "/dev/stdin", + ], + bytes(corrupted), + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertEqual(rejected.stdout, b"") + + @unittest.skipUnless( + os.environ.get("LABCOLORS_ARB_EVALUATOR"), + "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary", + ) + def test_multisegment_exact_trace_selects_first_canonical_branch(self) -> None: + registered = ContextualRegionDefinitionV1.parse( + (REPO / "proof/region/v1/fixtures/v5b2b-definition-0a8d1c3d.bin").read_bytes() + ) + zero = bytes(8) + tones = tuple( + bytes.fromhex(bits) + for bits in ( + "c000000000000000", + "bff0000000000000", + "0000000000000000", + "3ff0000000000000", + ) + ) + knots = tuple( + coordinate + for tone in tones + for coordinate in (tone, zero, zero, zero) + ) + definition = ContextualRegionDefinitionV1( + registered.fields[:21] + ((4).to_bytes(8, "big"),) + knots, + 4, + ) + job = ProofJobV1( + definition, + FORMULA.read_bytes(), + ReducedDomainManifestV1.from_ordinals((0,)), + ProofPolicyV1( + 1, + ( + ComparatorBudgetV1(ComparatorKindV1.ARB, (128,), 2, 2), + ComparatorBudgetV1(ComparatorKindV1.MPFI, (192,), 2, 2), + ), + ), + ) + manifest = ComparatorManifestV2( + ComparatorKindV1.ARB, + *(hashlib.sha256(f"arb-multisegment-{index}".encode()).digest() for index in range(10)), + ) + result = run_evaluator( + ( + os.environ["LABCOLORS_ARB_EVALUATOR"], + "--manifest-identity", + manifest.identity.hex(), + "--job", + "/dev/stdin", + ), + job.encode(), + ) + + self.assertEqual(result.returncode, 0, result.stderr.decode()) + transcript = DecisionTranscriptV1.parse(result.stdout) + self.assertEqual(tuple(transcript.iter_decisions()), (DecisionV1.INSIDE,)) + self.assertEqual(transcript.counters, (1, 0, 0, 0)) + self.assertEqual(transcript.exact_equality_count, 1) + witnesses = tuple(transcript.iter_witnesses()) + self.assertEqual(len(witnesses), 1) + self.assertIs(type(witnesses[0]), ExactZeroSignalTraceV1) + self.assertEqual( + witnesses[0].trace_digest, + hashlib.sha256( + b"labcolors.proof-region.exact-zero-signal-trace.v1\0" + + job.identity + + (0).to_bytes(4, "big") + + (1).to_bytes(8, "big") + ).digest(), + ) + + @unittest.skipUnless( + os.environ.get("LABCOLORS_ARB_EVALUATOR"), + "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary", + ) + def test_frozen_seam_cube_resolves_one_inside_and_511_outside(self) -> None: + frozen = ProofJobV1.parse( + (REPO / "proof/region/v1/fixtures/proof-job-v1.bin").read_bytes() + ) + budget = ( + ComparatorBudgetV1(ComparatorKindV1.ARB, (64, 128), 4, 2048), + ComparatorBudgetV1(ComparatorKindV1.MPFI, (64, 128), 4, 2048), + ) + job = ProofJobV1( + frozen.definition, + frozen.formula_spec, + frozen.domain, + ProofPolicyV1(1, budget), + ) + manifest = ComparatorManifestV2( + ComparatorKindV1.ARB, + *(hashlib.sha256(f"arb-manifest-{index}".encode()).digest() for index in range(10)), + ) + invocation = [ + os.environ["LABCOLORS_ARB_EVALUATOR"], + "--manifest-identity", + manifest.identity.hex(), + "--job", + "/dev/stdin", + ] + first = run_evaluator(invocation, job.encode()) + second = run_evaluator(invocation, job.encode()) + + self.assertEqual(first.returncode, 0, first.stderr.decode()) + self.assertEqual(second.returncode, 0, second.stderr.decode()) + self.assertEqual(first.stdout, second.stdout) + transcript = DecisionTranscriptV1.parse(first.stdout) + self.assertEqual(transcript.encode(), first.stdout) + assert_transcript_wire_coordinates( + self, + first.stdout, + transcript, + manifest.identity, + ) + self.assertEqual(transcript.job_identity, job.identity) + self.assertEqual(transcript.domain_identity, job.domain.identity) + self.assertEqual(transcript.comparator_identity, manifest.identity) + self.assertEqual(len(transcript.decision_bits), 128) + self.assertEqual(transcript.counters, (1, 511, 0, 0)) + self.assertEqual(transcript.exact_equality_count, 0) + self.assertEqual(tuple(transcript.iter_witnesses()), ()) + + low_precision = ProofJobV1( + frozen.definition, + frozen.formula_spec, + frozen.domain, + ProofPolicyV1( + 1, + ( + ComparatorBudgetV1(ComparatorKindV1.ARB, (16,), 4, 2_048), + ComparatorBudgetV1(ComparatorKindV1.MPFI, (24,), 4, 2_048), + ), + ), + ) + low_first = run_evaluator(invocation, low_precision.encode()) + low_second = run_evaluator(invocation, low_precision.encode()) + self.assertEqual(low_first.returncode, 0, low_first.stderr.decode()) + self.assertEqual(low_second.returncode, 0, low_second.stderr.decode()) + self.assertEqual(low_first.stdout, low_second.stdout) + low_transcript = DecisionTranscriptV1.parse(low_first.stdout) + self.assertEqual(low_transcript.encode(), low_first.stdout) + assert_transcript_wire_coordinates( + self, + low_first.stdout, + low_transcript, + manifest.identity, + ) + self.assertEqual(low_transcript.counters, (0, 501, 11, 0)) + low_witnesses = tuple(low_transcript.iter_witnesses()) + self.assertTrue( + all(type(witness) is BoundaryUnprovenWitnessV1 for witness in low_witnesses) + ) + self.assertEqual( + tuple(witness.ordinal for witness in low_witnesses), + ( + 65_793, + 657_930, + 723_723, + 8_355_711, + 8_421_247, + 8_421_503, + 8_421_504, + 16_711_422, + 16_776_958, + 16_777_214, + 16_777_215, + ), + ) + + @unittest.skipUnless( + os.environ.get("LABCOLORS_ARB_EVALUATOR"), + "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary", + ) + def test_zero_grant_emits_canonical_resource_witnesses(self) -> None: + frozen = ProofJobV1.parse( + (REPO / "proof/region/v1/fixtures/proof-job-v1.bin").read_bytes() + ) + zero_grant = ( + ComparatorBudgetV1(ComparatorKindV1.ARB, (64,), 0, 0), + ComparatorBudgetV1(ComparatorKindV1.MPFI, (64,), 0, 0), + ) + job = ProofJobV1( + frozen.definition, + frozen.formula_spec, + frozen.domain, + ProofPolicyV1(1, zero_grant), + ) + manifest = ComparatorManifestV2( + ComparatorKindV1.ARB, + *(hashlib.sha256(f"arb-zero-grant-{index}".encode()).digest() for index in range(10)), + ) + result = run_evaluator( + ( + os.environ["LABCOLORS_ARB_EVALUATOR"], + "--manifest-identity", + manifest.identity.hex(), + "--job", + "/dev/stdin", + ), + job.encode(), + ) + + self.assertEqual(result.returncode, 0, result.stderr.decode()) + transcript = DecisionTranscriptV1.parse(result.stdout) + self.assertEqual(transcript.encode(), result.stdout) + assert_transcript_wire_coordinates( + self, + result.stdout, + transcript, + manifest.identity, + ) + self.assertEqual(transcript.counters, (0, 504, 0, 8)) + witnesses = tuple(transcript.iter_witnesses()) + self.assertEqual( + tuple(witness.ordinal for witness in witnesses), + (10, 11, 256, 257, 65_537, 65_546, 65_792, 65_793), + ) + self.assertTrue(all(type(witness) is ResourceLimitWitnessV1 for witness in witnesses)) + self.assertTrue( + all( + (witness.scope, witness.granted, witness.consumed) == (1, 0, 0) + for witness in witnesses + ) + ) + + @unittest.skipUnless( + os.environ.get("LABCOLORS_ARB_EVALUATOR"), + "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary", + ) + def test_global_pregrant_is_never_transferred_between_points(self) -> None: + frozen = ProofJobV1.parse( + (REPO / "proof/region/v1/fixtures/proof-job-v1.bin").read_bytes() + ) + job = ProofJobV1( + frozen.definition, + frozen.formula_spec, + ReducedDomainManifestV1.from_ordinals((0, 65_793)), + ProofPolicyV1( + 1, + ( + ComparatorBudgetV1(ComparatorKindV1.ARB, (32,), 1, 1), + ComparatorBudgetV1(ComparatorKindV1.MPFI, (40,), 1, 1), + ), + ), + ) + manifest = ComparatorManifestV2( + ComparatorKindV1.ARB, + *(hashlib.sha256(f"arb-pregrant-{index}".encode()).digest() for index in range(10)), + ) + result = run_evaluator( + ( + os.environ["LABCOLORS_ARB_EVALUATOR"], + "--manifest-identity", + manifest.identity.hex(), + "--job", + "/dev/stdin", + ), + job.encode(), + ) + + self.assertEqual(result.returncode, 0, result.stderr.decode()) + transcript = DecisionTranscriptV1.parse(result.stdout) + self.assertEqual( + tuple(transcript.iter_decisions()), + (DecisionV1.OUTSIDE, DecisionV1.RESOURCE_LIMIT_REACHED), + ) + self.assertEqual(transcript.counters, (0, 1, 0, 1)) + witnesses = tuple(transcript.iter_witnesses()) + self.assertEqual( + witnesses, + (ResourceLimitWitnessV1(65_793, scope=2, granted=0, consumed=0),), + ) + + @unittest.skipUnless( + os.environ.get("LABCOLORS_ARB_EVALUATOR"), + "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary", + ) + def test_subminimum_precision_is_unresolved_and_a_later_valid_rung_recovers(self) -> None: + frozen = ProofJobV1.parse( + (REPO / "proof/region/v1/fixtures/proof-job-v1.bin").read_bytes() + ) + domain = ReducedDomainManifestV1.from_ordinals((0, 65_793)) + manifest = ComparatorManifestV2( + ComparatorKindV1.ARB, + *(hashlib.sha256(f"arb-minimum-precision-{index}".encode()).digest() for index in range(10)), + ) + invocation = ( + os.environ["LABCOLORS_ARB_EVALUATOR"], + "--manifest-identity", + manifest.identity.hex(), + "--job", + "/dev/stdin", + ) + + def run_with(arb_ladder: tuple[int, ...]) -> object: + job = ProofJobV1( + frozen.definition, + frozen.formula_spec, + domain, + ProofPolicyV1( + 1, + ( + ComparatorBudgetV1( + ComparatorKindV1.ARB, + arb_ladder, + 1, + 2, + ), + ComparatorBudgetV1( + ComparatorKindV1.MPFI, + (32,), + 1, + 2, + ), + ), + ), + ) + result = run_evaluator(invocation, job.encode()) + self.assertEqual(result.returncode, 0, result.stderr.decode()) + transcript = DecisionTranscriptV1.parse(result.stdout) + self.assertEqual(transcript.encode(), result.stdout) + return transcript + + unresolved = run_with((1,)) + direct = run_with((32,)) + recovered = run_with((1, 32)) + + self.assertEqual( + tuple(unresolved.iter_decisions()), + (DecisionV1.BOUNDARY_UNPROVEN, DecisionV1.BOUNDARY_UNPROVEN), + ) + self.assertEqual(unresolved.counters, (0, 0, 2, 0)) + self.assertEqual( + tuple(recovered.iter_decisions()), + tuple(direct.iter_decisions()), + ) + self.assertEqual(recovered.counters, direct.counters) + + @unittest.skipUnless( + os.environ.get("LABCOLORS_ARB_EVALUATOR"), + "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary", + ) + def test_resource_witness_accounts_for_work_consumed_on_earlier_rungs(self) -> None: + frozen = ProofJobV1.parse( + (REPO / "proof/region/v1/fixtures/proof-job-v1.bin").read_bytes() + ) + job = ProofJobV1( + frozen.definition, + frozen.formula_spec, + ReducedDomainManifestV1.from_ordinals((257,)), + ProofPolicyV1( + 1, + ( + ComparatorBudgetV1(ComparatorKindV1.ARB, (12, 64), 1, 1), + ComparatorBudgetV1(ComparatorKindV1.MPFI, (20, 80), 1, 1), + ), + ), + ) + manifest = ComparatorManifestV2( + ComparatorKindV1.ARB, + *(hashlib.sha256(f"arb-cross-rung-{index}".encode()).digest() for index in range(10)), + ) + result = run_evaluator( + ( + os.environ["LABCOLORS_ARB_EVALUATOR"], + "--manifest-identity", + manifest.identity.hex(), + "--job", + "/dev/stdin", + ), + job.encode(), + ) + + self.assertEqual(result.returncode, 0, result.stderr.decode()) + transcript = DecisionTranscriptV1.parse(result.stdout) + self.assertEqual( + tuple(transcript.iter_decisions()), + (DecisionV1.RESOURCE_LIMIT_REACHED,), + ) + self.assertEqual(transcript.counters, (0, 0, 0, 1)) + self.assertEqual( + tuple(transcript.iter_witnesses()), + (ResourceLimitWitnessV1(257, scope=1, granted=1, consumed=1),), + ) + + @unittest.skipUnless( + os.environ.get("LABCOLORS_ARB_EVALUATOR"), + "set LABCOLORS_ARB_EVALUATOR to the controlled C17 binary", + ) + def test_spd_admission_is_exact_across_the_full_binary64_exponent_range(self) -> None: + frozen = ProofJobV1.parse( + (REPO / "proof/region/v1/fixtures/proof-job-v1.bin").read_bytes() + ) + fields = list(frozen.definition.fields) + fields[18] = bytes.fromhex("3ff0000000000000") + fields[19] = bytes.fromhex("0000000000000001") + fields[20] = bytes.fromhex("3ff0000000000000") + definition = ContextualRegionDefinitionV1( + tuple(fields), + frozen.definition.knot_count, + ) + job = ProofJobV1( + definition, + frozen.formula_spec, + ReducedDomainManifestV1.from_ordinals((0,)), + ProofPolicyV1( + 1, + ( + ComparatorBudgetV1(ComparatorKindV1.ARB, (64,), 4, 4), + ComparatorBudgetV1(ComparatorKindV1.MPFI, (80,), 4, 4), + ), + ), + ) + manifest = ComparatorManifestV2( + ComparatorKindV1.ARB, + *(hashlib.sha256(f"arb-exact-spd-{index}".encode()).digest() for index in range(10)), + ) + result = run_evaluator( + ( + os.environ["LABCOLORS_ARB_EVALUATOR"], + "--manifest-identity", + manifest.identity.hex(), + "--job", + "/dev/stdin", + ), + job.encode(), + ) + + self.assertEqual(result.returncode, 0, result.stderr.decode()) + transcript = DecisionTranscriptV1.parse(result.stdout) + self.assertEqual(transcript.encode(), result.stdout) + self.assertEqual(transcript.job_identity, job.identity) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/proof/region/v1/arb/tests/test_executor.py b/proof/region/v1/arb/tests/test_executor.py new file mode 100644 index 00000000..8be7b4a4 --- /dev/null +++ b/proof/region/v1/arb/tests/test_executor.py @@ -0,0 +1,1510 @@ +#!/usr/bin/env python3 +"""Hostile tests for the Linux-only Arb process boundary.""" + +from __future__ import annotations + +import errno +import fcntl +import hashlib +import os +import signal +import struct +import sys +import tempfile +import threading +import unittest +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from pathlib import Path +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[2] +ARB_ROOT = ROOT / "arb" +sys.path.insert(0, str(ARB_ROOT)) + +import executor # noqa: E402 + + +def _static_elf(*, interpreter: bool = False, needed: bool = False) -> bytes: + """Return a parseable ELF64/x86-64 shape; it is not intended to run.""" + + program_headers: list[bytes] = [] + body = bytearray(64) + program_headers.append( + struct.pack(" tuple[tuple[int, ...], ...]: + count = struct.unpack_from(" bytes: + """Create a literal static ELF64 fixture with one RX load segment.""" + + code_offset = 64 + 56 + file_size = code_offset + len(code) + ident = b"\x7fELF\x02\x01\x01" + bytes(9) + header = ident + struct.pack( + " executor.ExecutionLimitsV1: + values = { + "max_executable_bytes": 4096, + "max_stdin_bytes": 4096, + "max_argument_bytes": 4096, + "max_stdout_bytes": 16, + "max_stderr_bytes": 16, + "wall_timeout_ns": 1_000_000_000, + "memory_max_bytes": 64 * 1024 * 1024, + "pids_max": 1, + } + values.update(changes) + return executor.ExecutionLimitsV1(**values) + + +def _request(**changes: object) -> executor.ExecutionRequestV1: + values: dict[str, object] = { + "executable": _static_elf(), + "argv": ( + b"arb-evaluator", + b"--manifest-identity", + b"1" * 64, + b"--job", + b"/dev/stdin", + ), + "environment": ((b"LC_ALL", b"C"), (b"TZ", b"UTC")), + "cwd": b"/work", + "stdin": b"LCJOB1\0\0", + "umask": 0o077, + "limits": _limits(), + } + values.update(changes) + return executor.ExecutionRequestV1(**values) + + +class _Backend: + def __init__( + self, + probe_result: executor.CapabilityReportV1, + run_result: executor.ExecutionResultV1 | None = None, + ) -> None: + self.probe_result = probe_result + self.run_result = run_result + self.probe_calls = 0 + self.received: list[ + tuple[executor.ExecutionRequestV1, executor.SupportedV1] + ] = [] + + def probe(self, guard: object) -> executor.CapabilityReportV1: + self.probe_calls += 1 + if not guard.is_current(): # type: ignore[attr-defined] + raise AssertionError("controller supplied a stale probe guard") + return self.probe_result + + def run( + self, + request: executor.ExecutionRequestV1, + capability: executor.SupportedV1, + ) -> executor.ExecutionResultV1: + self.received.append((request, capability)) + if self.run_result is None: + raise AssertionError("unsupported backend must not be run") + return self.run_result + + +class _MemfdOperations: + def __init__(self) -> None: + self.fd = 41 + self.bytes_by_fd: dict[int, bytes] = {} + self.seals_by_fd: dict[int, int] = {} + self.exec_calls: list[tuple[int, tuple[bytes, ...], tuple[tuple[bytes, bytes], ...]]] = [] + self.events: list[str] = [] + + def create_executable_memfd(self) -> int: + self.events.append("create") + return self.fd + + def pipe_cloexec(self) -> tuple[int, int]: + read_fd, write_fd = os.pipe() + for descriptor in (read_fd, write_fd): + flags = fcntl.fcntl(descriptor, fcntl.F_GETFD) + fcntl.fcntl(descriptor, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) + return read_fd, write_fd + + def write_all(self, fd: int, data: bytes) -> None: + self.events.append("write") + self.bytes_by_fd[fd] = data + + def make_executable(self, fd: int) -> None: + self.events.append("chmod") + self.assert_known(fd) + + def add_seals(self, fd: int, seals: int) -> None: + self.events.append("seal") + self.assert_known(fd) + self.seals_by_fd[fd] = seals + + def get_seals(self, fd: int) -> int: + self.events.append("get_seals") + self.assert_known(fd) + return self.seals_by_fd[fd] + + def pread(self, fd: int, size: int, offset: int) -> bytes: + self.events.append("pread") + self.assert_known(fd) + return self.bytes_by_fd[fd][offset : offset + size] + + def execveat( + self, + fd: int, + argv: tuple[bytes, ...], + environment: tuple[tuple[bytes, bytes], ...], + ) -> None: + self.events.append("execveat") + self.assert_known(fd) + self.exec_calls.append((fd, argv, environment)) + + def close(self, fd: int) -> None: + self.assert_known(fd) + + def assert_known(self, fd: int) -> None: + if fd != self.fd: + raise AssertionError(f"unexpected file descriptor: {fd}") + + +class _ProbeOperations(_MemfdOperations): + def __init__(self) -> None: + super().__init__() + self.probes: list[str] = [] + + def probe_execveat(self) -> None: + self.probes.append("execveat") + + def probe_standard_fds(self) -> None: + self.probes.append("standard_fds") + + def probe_single_threaded(self) -> None: + self.probes.append("single_threaded") + + def probe_close_range(self) -> None: + self.probes.append("close_range") + + def probe_namespaces(self) -> None: + self.probes.append("namespaces") + + def probe_seccomp(self) -> None: + self.probes.append("seccomp") + + +class _LateThreadOperations(_ProbeOperations): + def probe_single_threaded(self) -> None: + self.probes.append("single_threaded") + raise OSError(errno.EBUSY, "late thread") + + +class _OverlapAfterSingleThreadOperations(_ProbeOperations): + def __init__(self) -> None: + super().__init__() + self.single_thread_passed = threading.Event() + self.release_outer_probe = threading.Event() + self.blocked_once = False + + def probe_single_threaded(self) -> None: + super().probe_single_threaded() + if not self.blocked_once: + self.blocked_once = True + self.single_thread_passed.set() + if not self.release_outer_probe.wait(timeout=1): + raise AssertionError("overlap test did not release the outer probe") + + +class _SecondSingleThreadFailureOperations(_ProbeOperations): + def __init__(self) -> None: + super().__init__() + self.single_thread_probes = 0 + + def probe_single_threaded(self) -> None: + super().probe_single_threaded() + self.single_thread_probes += 1 + if self.single_thread_probes == 2: + raise OSError(errno.EBUSY, "thread appeared before fork") + + +class _CgroupFactory: + def __init__(self) -> None: + self.observer_budgets: list[Path] = [] + self.probed: list[Path] = [] + + def probe_observer_task_budget(self, parent: Path) -> None: + self.observer_budgets.append(parent) + + def probe(self, parent: Path) -> None: + self.probed.append(parent) + + +class _ObserverCgroup: + def __init__(self, pid: int) -> None: + self.pid = pid + + def kill_all(self) -> None: + try: + os.kill(self.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def oom_kill_count(self) -> int: + return 0 + + def populated(self) -> bool: + return False + + +class _ReadbackCgroup(executor._CgroupV2V1): + def __init__(self, values: dict[bytes, bytes]) -> None: + self.values = values + + def _read_required(self, name: bytes) -> bytes: + return self.values[name] + + +class RequestAdmissionTests(unittest.TestCase): + def test_combined_dynamic_fixture_points_after_its_full_header_table(self) -> None: + elf = _static_elf(interpreter=True, needed=True) + headers = _program_headers(elf) + dynamic = next(header for header in headers if header[0] == 2) + + self.assertEqual(dynamic[2], 64 + 56 * len(headers)) + self.assertEqual( + elf[dynamic[2] : dynamic[2] + dynamic[5]], + struct.pack(" None: + with self.assertRaises(executor.ExecutionRequestErrorV1) as caught: + _request(**changes) + self.assertEqual(caught.exception.reason, reason) + + def test_request_preserves_exact_invocation_without_mapping_or_inheritance(self) -> None: + request = _request() + + self.assertEqual( + request.argv, + ( + b"arb-evaluator", + b"--manifest-identity", + b"1" * 64, + b"--job", + b"/dev/stdin", + ), + ) + self.assertEqual(request.environment, ((b"LC_ALL", b"C"), (b"TZ", b"UTC"))) + self.assertEqual(request.cwd, b"/work") + self.assertEqual(request.stdin, b"LCJOB1\0\0") + self.assertNotIn("network_isolated", request.__dataclass_fields__) + self.assertNotIn("cgroup_isolated", request.__dataclass_fields__) + + def test_argv_environment_cwd_and_stdin_are_strict_bytes(self) -> None: + cases = ( + ({"argv": [b"arb-evaluator"]}, executor.RequestReasonV1.WRONG_TYPE), + ({"argv": (b"",)}, executor.RequestReasonV1.EMPTY_ARGV_ZERO), + ({"argv": (b"arb\0evil",)}, executor.RequestReasonV1.NUL_BYTE), + ({"environment": {b"LC_ALL": b"C"}}, executor.RequestReasonV1.WRONG_TYPE), + ( + {"environment": ((b"TZ", b"UTC"), (b"LC_ALL", b"C"))}, + executor.RequestReasonV1.NONCANONICAL_ENVIRONMENT, + ), + ( + {"environment": ((b"LC_ALL", b"C"), (b"LC_ALL", b"POSIX"))}, + executor.RequestReasonV1.DUPLICATE_ENVIRONMENT, + ), + ({"environment": ((b"A=B", b"C"),)}, executor.RequestReasonV1.INVALID_ENVIRONMENT_KEY), + ({"cwd": b"relative"}, executor.RequestReasonV1.RELATIVE_CWD), + ({"cwd": b"/work/../tmp"}, executor.RequestReasonV1.NONCANONICAL_CWD), + ({"stdin": "not-bytes"}, executor.RequestReasonV1.WRONG_TYPE), + ) + for changes, reason in cases: + with self.subTest(changes=changes): + self.assert_rejected(reason, **changes) + + def test_explicit_limits_reject_oversized_inputs_and_bool_numbers(self) -> None: + self.assert_rejected( + executor.RequestReasonV1.LIMIT_EXCEEDED, + stdin=b"12345", + limits=_limits(max_stdin_bytes=4), + ) + self.assert_rejected( + executor.RequestReasonV1.LIMIT_EXCEEDED, + executable=_static_elf() + b"x" * 4096, + ) + with self.assertRaises(executor.ExecutionRequestErrorV1) as caught: + replace(_limits(), pids_max=True) # type: ignore[arg-type] + self.assertEqual(caught.exception.reason, executor.RequestReasonV1.INVALID_LIMIT) + with self.assertRaises(executor.ExecutionRequestErrorV1) as caught: + replace(_limits(), pids_max=2) + self.assertEqual(caught.exception.reason, executor.RequestReasonV1.INVALID_LIMIT) + + def test_only_static_x86_64_elf_is_admitted(self) -> None: + self.assert_rejected(executor.RequestReasonV1.INVALID_ELF, executable=b"#!/bin/sh\n") + self.assert_rejected( + executor.RequestReasonV1.DYNAMIC_EXECUTABLE, + executable=_static_elf(interpreter=True), + ) + self.assert_rejected( + executor.RequestReasonV1.DYNAMIC_EXECUTABLE, + executable=_static_elf(needed=True), + ) + + def test_cross_module_verifiers_are_explicit_versioned_api(self) -> None: + self.assertTrue(callable(executor.require_static_x86_64_elf_v1)) + self.assertTrue(callable(executor.result_matches_request_v1)) + self.assertFalse(hasattr(executor, "_require_static_x86_64_elf")) + self.assertFalse(hasattr(executor, "_result_matches_request")) + + +class CapabilityAndExecutionTests(unittest.TestCase): + def test_non_linux_host_fails_closed_before_any_run(self) -> None: + native = executor.NativeLinuxBackendV1( + cgroup_parent=None, + platform_name="darwin", + machine_name="arm64", + ) + controller = executor.ControlledExecutorV1(native) + report = controller.probe() + + self.assertIs(type(report), executor.UnsupportedV1) + self.assertEqual( + report.failures, + ( + executor.CapabilityFailureV1( + executor.CapabilityReasonV1.HOST_NOT_LINUX, + None, + ), + ), + ) + result = controller.execute(_request()) + self.assertEqual(result, report) + + def test_linux_without_an_explicit_delegated_cgroup_is_unsupported(self) -> None: + native = executor.NativeLinuxBackendV1( + cgroup_parent=None, + platform_name="linux", + machine_name="x86_64", + ) + report = executor.ControlledExecutorV1(native).probe() + + self.assertIs(type(report), executor.UnsupportedV1) + self.assertIn( + executor.CapabilityFailureV1( + executor.CapabilityReasonV1.CGROUP_PARENT_NOT_DECLARED, + None, + ), + report.failures, + ) + + def test_supported_probe_executes_every_required_mechanism(self) -> None: + operations = _ProbeOperations() + cgroups = _CgroupFactory() + native = executor.NativeLinuxBackendV1( + cgroup_parent="/delegated-proof-cgroup", + platform_name="linux", + machine_name="x86_64", + operations=operations, + cgroup_factory=cgroups, + ) + + report = executor.ControlledExecutorV1(native).probe() + + self.assertEqual( + report, + executor.SupportedV1( + "linux-x86_64", + executor.SANDBOX_POLICY_RELEASE_V1, + ), + ) + self.assertEqual( + operations.probes, + [ + "standard_fds", + "single_threaded", + "execveat", + "close_range", + "single_threaded", + "namespaces", + "single_threaded", + "seccomp", + ], + ) + self.assertEqual( + cgroups.observer_budgets, + [Path("/delegated-proof-cgroup")], + ) + self.assertEqual(cgroups.probed, [Path("/delegated-proof-cgroup")]) + + def test_final_probe_cannot_outlive_its_controller_lease(self) -> None: + current = True + + class InvalidatingCgroupFactory(_CgroupFactory): + def probe(self, parent: Path) -> None: + nonlocal current + super().probe(parent) + current = False + + native = executor.NativeLinuxBackendV1( + cgroup_parent="/delegated-proof-cgroup", + platform_name="linux", + machine_name="x86_64", + operations=_ProbeOperations(), + cgroup_factory=InvalidatingCgroupFactory(), + ) + + report = native._probe_capability_v1( + executor._ProbeGuardV1(lambda: current) + ) + + self.assertEqual(report, executor._invalidated_capability_report_v1()) + + def test_overlap_after_single_thread_gate_cancels_before_fork_and_revokes_authority(self) -> None: + operations = _OverlapAfterSingleThreadOperations() + native = executor.NativeLinuxBackendV1( + cgroup_parent="/delegated-proof-cgroup", + platform_name="linux", + machine_name="x86_64", + operations=operations, + cgroup_factory=_CgroupFactory(), + ) + controller = executor.ControlledExecutorV1(native) + reports: list[executor.CapabilityReportV1] = [] + worker = threading.Thread( + target=lambda: reports.append(controller.probe()), + daemon=True, + ) + + worker.start() + self.assertTrue( + operations.single_thread_passed.wait(timeout=1), + "outer probe did not reach the single-thread gate", + ) + overlap = controller.probe() + operations.release_outer_probe.set() + worker.join(timeout=1) + + self.assertFalse(worker.is_alive(), "overlapping probe deadlocked") + self.assertEqual(len(reports), 1) + invalidated = executor.UnsupportedV1( + ( + executor.CapabilityFailureV1( + executor.CapabilityReasonV1.OBSERVATION_INVALIDATED, + errno.EBUSY, + ), + ) + ) + self.assertEqual(overlap, invalidated) + self.assertEqual(reports[0], invalidated) + self.assertEqual(operations.probes, ["standard_fds", "single_threaded"]) + self.assertIsNone(controller._issued_capability) + self.assertEqual( + controller.probe(), + executor.SupportedV1( + "linux-x86_64", + executor.SANDBOX_POLICY_RELEASE_V1, + ), + ) + + def test_failed_single_thread_gate_suppresses_every_forking_probe(self) -> None: + operations = _LateThreadOperations() + cgroups = _CgroupFactory() + native = executor.NativeLinuxBackendV1( + cgroup_parent="/delegated-proof-cgroup", + platform_name="linux", + machine_name="x86_64", + operations=operations, + cgroup_factory=cgroups, + ) + + report = executor.ControlledExecutorV1(native).probe() + + self.assertEqual( + report, + executor.UnsupportedV1( + ( + executor.CapabilityFailureV1( + executor.CapabilityReasonV1.OBSERVER_NOT_SINGLE_THREADED, + errno.EBUSY, + ), + ) + ), + ) + self.assertEqual(operations.probes, ["standard_fds", "single_threaded"]) + self.assertEqual(cgroups.observer_budgets, []) + self.assertEqual(cgroups.probed, []) + + def test_second_single_thread_gate_suppresses_forking_probe(self) -> None: + operations = _SecondSingleThreadFailureOperations() + native = executor.NativeLinuxBackendV1( + cgroup_parent="/delegated-proof-cgroup", + platform_name="linux", + machine_name="x86_64", + operations=operations, + cgroup_factory=_CgroupFactory(), + ) + + report = executor.ControlledExecutorV1(native).probe() + + self.assertEqual( + report, + executor.UnsupportedV1( + ( + executor.CapabilityFailureV1( + executor.CapabilityReasonV1.OBSERVER_NOT_SINGLE_THREADED, + errno.EBUSY, + ), + ) + ), + ) + self.assertEqual( + operations.probes, + ["standard_fds", "single_threaded", "execveat", "close_range", "single_threaded"], + ) + + def test_one_probe_capability_is_forwarded_to_exactly_one_run(self) -> None: + capability = executor.SupportedV1( + "linux-x86_64", + executor.SANDBOX_POLICY_RELEASE_V1, + ) + expected = executor.CompletedV1( + binary_sha256=hashlib.sha256(_static_elf()).digest(), + stdout=b"answer", + stderr=b"", + ) + backend = _Backend(capability, expected) + request = _request() + + actual = executor.ControlledExecutorV1(backend).execute(request) + + self.assertEqual(actual, expected) + self.assertEqual(backend.probe_calls, 1) + self.assertEqual(len(backend.received), 1) + received_request, received_capability = backend.received[0] + self.assertIs(received_request, request) + self.assertEqual(received_capability, capability) + self.assertIsNot(received_capability, capability) + + def test_preprobed_capability_is_consumed_without_a_second_probe(self) -> None: + capability = executor.SupportedV1( + "linux-x86_64", + executor.SANDBOX_POLICY_RELEASE_V1, + ) + request = _request() + expected = executor.CompletedV1( + binary_sha256=hashlib.sha256(request.executable).digest(), + stdout=b"answer", + stderr=b"", + ) + backend = _Backend(capability, expected) + controller = executor.ControlledExecutorV1(backend) + issued = controller.probe() + self.assertEqual(issued, capability) + self.assertIsNot(issued, capability) + + with mock.patch.object( + backend, + "probe", + side_effect=AssertionError("execute must consume the supplied observation"), + ): + actual = controller.execute(request, issued) + + self.assertEqual(actual, expected) + self.assertEqual(backend.probe_calls, 1) + self.assertEqual( + controller.execute(request, issued), + executor.ObserverFailureV1(executor.ObserverReasonV1.PROBE_FAILED), + ) + + def test_backend_reused_report_cannot_renew_a_stale_controller_lease(self) -> None: + backend_report = executor.SupportedV1( + "linux-x86_64", + executor.SANDBOX_POLICY_RELEASE_V1, + ) + request = _request() + expected = executor.CompletedV1( + binary_sha256=hashlib.sha256(request.executable).digest(), + stdout=b"answer", + stderr=b"", + ) + backend = _Backend(backend_report, expected) + controller = executor.ControlledExecutorV1(backend) + + stale = controller.probe() + fresh = controller.probe() + + self.assertIs(type(stale), executor.SupportedV1) + self.assertIs(type(fresh), executor.SupportedV1) + self.assertIsNot(stale, fresh) + self.assertEqual( + controller.execute(request, stale), + executor.ObserverFailureV1(executor.ObserverReasonV1.PROBE_FAILED), + ) + self.assertEqual(backend.received, []) + self.assertEqual(controller.execute(request, fresh), expected) + self.assertEqual(len(backend.received), 1) + + def test_controller_capability_cannot_be_duplicated_across_fork(self) -> None: + backend_report = executor.SupportedV1( + "linux-x86_64", + executor.SANDBOX_POLICY_RELEASE_V1, + ) + request = _request() + expected = executor.CompletedV1( + binary_sha256=hashlib.sha256(request.executable).digest(), + stdout=b"answer", + stderr=b"", + ) + backend = _Backend(backend_report, expected) + controller = executor.ControlledExecutorV1(backend) + capability = controller.probe() + read_descriptor, write_descriptor = os.pipe() + + child = os.fork() + if child == 0: + os.close(read_descriptor) + try: + result = controller.execute(request, capability) + payload = ( + b"blocked" + if result + == executor.ObserverFailureV1( + executor.ObserverReasonV1.PROBE_FAILED + ) + else b"executed" + ) + os.write(write_descriptor, payload) + status = 0 + except BaseException: + status = 1 + finally: + os.close(write_descriptor) + os._exit(status) + + os.close(write_descriptor) + try: + payload = os.read(read_descriptor, 32) + finally: + os.close(read_descriptor) + waited, status = os.waitpid(child, 0) + + self.assertEqual(waited, child) + self.assertTrue(os.WIFEXITED(status)) + self.assertEqual(os.WEXITSTATUS(status), 0) + self.assertEqual(payload, b"blocked") + self.assertEqual(controller.execute(request, capability), expected) + self.assertEqual(len(backend.received), 1) + + def test_capability_cannot_cross_a_backend_replacement(self) -> None: + request = _request() + capability = executor.SupportedV1( + "linux-x86_64", + executor.SANDBOX_POLICY_RELEASE_V1, + ) + expected = executor.CompletedV1( + binary_sha256=hashlib.sha256(request.executable).digest(), + stdout=b"answer", + stderr=b"", + ) + original = _Backend(capability, expected) + replacement = _Backend(capability, expected) + controller = executor.ControlledExecutorV1(original) + issued = controller.probe() + self.assertEqual(issued, capability) + self.assertIsNot(issued, capability) + controller._backend = replacement + + result = controller.execute(request, issued) + + self.assertEqual( + result, + executor.ObserverFailureV1(executor.ObserverReasonV1.PROBE_FAILED), + ) + self.assertEqual(original.received, []) + self.assertEqual(replacement.received, []) + + def test_native_run_consumes_capability_without_reprobe_and_rejects_foreign(self) -> None: + operations = _ProbeOperations() + native = executor.NativeLinuxBackendV1( + cgroup_parent="/delegated-proof-cgroup", + platform_name="linux", + machine_name="x86_64", + operations=operations, + cgroup_factory=_CgroupFactory(), + ) + controller = executor.ControlledExecutorV1(native) + capability = controller.probe() + self.assertIs(type(capability), executor.SupportedV1) + creates_after_probe = operations.events.count("create") + request = _request(cwd=b"/definitely-missing-labcolors-cwd") + + with mock.patch.object( + native, + "probe", + side_effect=AssertionError("run must consume, not repeat, capability probe"), + ): + result = controller.execute(request, capability) + + self.assertIs(type(result), executor.SandboxSetupFailedV1) + self.assertEqual(result.stage, executor.SetupStageV1.CWD) + self.assertEqual(operations.events.count("create"), creates_after_probe + 1) + + equal_but_foreign = executor.SupportedV1( + capability.platform, + capability.sandbox_policy_release, + ) + self.assertEqual(equal_but_foreign, capability) + self.assertIsNot(equal_but_foreign, capability) + for invalid in (capability, equal_but_foreign, object()): + with self.subTest(invalid=invalid): + with mock.patch.object( + executor, + "_seal_executable_v1", + side_effect=AssertionError("foreign capability must not execute"), + ): + rejected = controller.execute( # type: ignore[arg-type] + request, + invalid, + ) + self.assertEqual( + rejected, + executor.ObserverFailureV1( + executor.ObserverReasonV1.PROBE_FAILED, + ), + ) + + def test_native_capability_has_one_atomic_consumer(self) -> None: + operations = _ProbeOperations() + native = executor.NativeLinuxBackendV1( + cgroup_parent="/delegated-proof-cgroup", + platform_name="linux", + machine_name="x86_64", + operations=operations, + cgroup_factory=_CgroupFactory(), + ) + controller = executor.ControlledExecutorV1(native) + capability = controller.probe() + self.assertIs(type(capability), executor.SupportedV1) + creates_after_probe = operations.events.count("create") + request = _request(cwd=b"/definitely-missing-labcolors-cwd") + + with ThreadPoolExecutor(max_workers=2) as pool: + results = tuple( + pool.map( + lambda _index: controller.execute(request, capability), + range(2), + ) + ) + + self.assertEqual(operations.events.count("create"), creates_after_probe + 1) + self.assertEqual( + sum(type(result) is executor.SandboxSetupFailedV1 for result in results), + 1, + ) + self.assertEqual( + sum( + result + == executor.ObserverFailureV1(executor.ObserverReasonV1.PROBE_FAILED) + for result in results + ), + 1, + ) + + def test_failed_native_probe_revokes_earlier_capability(self) -> None: + native = executor.NativeLinuxBackendV1( + cgroup_parent="/delegated-proof-cgroup", + platform_name="linux", + machine_name="x86_64", + operations=_ProbeOperations(), + cgroup_factory=_CgroupFactory(), + ) + controller = executor.ControlledExecutorV1(native) + stale = controller.probe() + self.assertIs(type(stale), executor.SupportedV1) + native._platform_name = "darwin" + + failed = controller.probe() + + self.assertIs(type(failed), executor.UnsupportedV1) + with mock.patch.object( + executor, + "_seal_executable_v1", + side_effect=AssertionError("failed probe must revoke earlier capability"), + ): + rejected = controller.execute(_request(), stale) + self.assertEqual( + rejected, + executor.ObserverFailureV1(executor.ObserverReasonV1.PROBE_FAILED), + ) + + def test_untrusted_backend_cannot_return_unbounded_output(self) -> None: + class ExplosiveEquality: + def __eq__(self, _other: object) -> bool: + raise RuntimeError("comparison escaped") + + invalid_results = ( + executor.CompletedV1( + binary_sha256=b"x" * 32, + stdout=b"17 bytes overflow", + stderr=b"", + ), + executor.CompletedV1(ExplosiveEquality(), b"", b""), + executor.ObserverFailureV1(ExplosiveEquality()), + ) + for invalid in invalid_results: + with self.subTest(invalid=type(invalid).__name__): + self.assertFalse(executor.result_matches_request_v1(invalid, _request())) + backend = _Backend( + executor.SupportedV1( + "linux-x86_64", + executor.SANDBOX_POLICY_RELEASE_V1, + ), + invalid, + ) + + result = executor.ControlledExecutorV1(backend).execute(_request()) + + self.assertIs(type(result), executor.ObserverFailureV1) + self.assertEqual( + result.reason, + executor.ObserverReasonV1.BACKEND_CONTRACT, + ) + self.assertFalse(hasattr(result, "stdout")) + + def test_process_failures_remain_distinct_from_evaluator_resource_outcome(self) -> None: + digest = hashlib.sha256(_static_elf()).digest() + results: tuple[executor.ExecutionResultV1, ...] = ( + executor.ExitNonZeroV1(digest, b"", b"bad", 17), + executor.SignaledV1(digest, b"", b"", 11, True), + executor.TimedOutV1(digest, b"", b"", 1_000_000_000), + executor.OomKilledV1(digest, b"", b"", 1), + executor.OutputLimitExceededV1( + digest, + b"x" * 16, + b"", + executor.OutputStreamV1.STDOUT, + 16, + ), + ) + for expected in results: + with self.subTest(result_type=type(expected).__name__): + backend = _Backend( + executor.SupportedV1( + "linux-x86_64", executor.SANDBOX_POLICY_RELEASE_V1 + ), + expected, + ) + actual = executor.ControlledExecutorV1(backend).execute(_request()) + self.assertEqual(actual, expected) + self.assertNotIn("ResourceLimit", type(actual).__name__) + + def test_executor_exports_observations_but_no_receipt_mint(self) -> None: + self.assertFalse(any("Receipt" in name for name in dir(executor))) + self.assertFalse(hasattr(executor.ControlledExecutorV1, "mint")) + self.assertFalse(hasattr(executor.ControlledExecutorV1, "admit")) + + +class SameObjectAndObserverProtocolTests(unittest.TestCase): + @staticmethod + def _seccomp_verdict( + program: list[object], + syscall_number: int, + *, + architecture: int = 0xC000003E, + arguments: tuple[int, ...] = (0, 0, 0, 0, 0, 0), + ) -> int: + words = {0: syscall_number, 4: architecture} + for index, argument in enumerate(arguments): + words[16 + index * 8] = argument & 0xFFFFFFFF + words[20 + index * 8] = (argument >> 32) & 0xFFFFFFFF + accumulator = 0 + pc = 0 + for _ in range(1024): + instruction = program[pc] + if instruction.code == 0x20: # BPF_LD | BPF_W | BPF_ABS + accumulator = words.get(instruction.k, 0) + pc += 1 + elif instruction.code == 0x15: # BPF_JMP | BPF_JEQ | BPF_K + pc += 1 + (instruction.jt if accumulator == instruction.k else instruction.jf) + elif instruction.code == 0x06: # BPF_RET | BPF_K + return instruction.k + else: + raise AssertionError(f"unknown BPF opcode {instruction.code:#x}") + raise AssertionError("seccomp program did not terminate") + + def test_seccomp_filter_denies_files_network_processes_and_exec_path_swaps(self) -> None: + operations = executor._NativeLinuxOperationsV1() + program = operations._seccomp_program(exec_fd=3, setup_error_fd=4) + killed = 0x80000000 + allowed = 0x7FFF0000 + + self.assertEqual(self._seccomp_verdict(program, 1), allowed) # write + for syscall_number in (2, 41, 56, 257, 319): + with self.subTest(syscall_number=syscall_number): + self.assertEqual(self._seccomp_verdict(program, syscall_number), killed) + self.assertEqual( + self._seccomp_verdict(program, 302, arguments=(0, 0, 0, 0, 0, 0)), + allowed, + ) + for foreign_pid in (1, 42, 0xFFFFFFFFFFFFFFFF): + with self.subTest(prlimit_pid=foreign_pid): + self.assertEqual( + self._seccomp_verdict( + program, + 302, + arguments=(foreign_pid, 0, 0, 0, 0, 0), + ), + killed, + ) + self.assertEqual( + self._seccomp_verdict( + program, + 322, + arguments=(3, 0, 0, 0, 0x1000, 0), + ), + allowed, + ) + self.assertEqual( + self._seccomp_verdict( + program, + 322, + arguments=(5, 0, 0, 0, 0x1000, 0), + ), + killed, + ) + self.assertEqual( + self._seccomp_verdict( + program, + 322, + arguments=(3, 0, 0, 0, 0, 0), + ), + killed, + ) + self.assertEqual( + self._seccomp_verdict(program, 1, architecture=0x40000003), + killed, + ) + + def test_hash_and_exec_use_the_same_sealed_memfd(self) -> None: + operations = _MemfdOperations() + executable = _static_elf() + + sealed = executor._seal_executable_v1(executable, operations) + sealed.execveat( + (b"arb-evaluator",), + ((b"LC_ALL", b"C"),), + operations, + ) + + self.assertEqual(sealed.fd, operations.fd) + self.assertEqual(sealed.sha256, hashlib.sha256(executable).digest()) + self.assertEqual( + operations.seals_by_fd[sealed.fd] & executor.REQUIRED_FILE_SEALS_V1, + executor.REQUIRED_FILE_SEALS_V1, + ) + self.assertEqual(operations.exec_calls[0][0], sealed.fd) + self.assertEqual( + operations.events, + ["create", "write", "chmod", "seal", "get_seals", "pread", "execveat"], + ) + + def test_child_error_packet_rejects_unknown_trailing_and_truncated_bytes(self) -> None: + valid = executor._encode_child_error_packet_v1( + executor.SetupStageV1.EXECVEAT, + 8, + ) + parsed = executor._parse_child_error_packet_v1(valid) + self.assertEqual(parsed.stage, executor.SetupStageV1.EXECVEAT) + self.assertEqual(parsed.errno, 8) + + cases = ( + b"BAD!" + valid[4:], + valid[:-1], + valid + b"\0", + valid[:5] + b"\xff" + valid[6:], + ) + for packet in cases: + with self.subTest(packet=packet): + with self.assertRaises(executor.ObserverProtocolErrorV1): + executor._parse_child_error_packet_v1(packet) + + def test_capture_keeps_exact_cap_and_detects_only_cap_plus_one(self) -> None: + captured = bytearray() + + self.assertFalse(executor._append_bounded_v1(captured, b"1234", 4)) + self.assertEqual(bytes(captured), b"1234") + self.assertTrue(executor._append_bounded_v1(captured, b"5", 4)) + self.assertEqual(bytes(captured), b"1234") + + def test_observer_does_not_infer_oom_from_sigkill(self) -> None: + digest = hashlib.sha256(_static_elf()).digest() + + signal_only = executor._classify_process_v1( + digest=digest, + stdout=b"", + stderr=b"", + child_status=9, + oom_kill_delta=0, + residual=False, + setup_packet=b"", + terminal=None, + limits=_limits(), + ) + actual_oom = executor._classify_process_v1( + digest=digest, + stdout=b"", + stderr=b"", + child_status=9, + oom_kill_delta=2, + residual=False, + setup_packet=b"", + terminal=None, + limits=_limits(), + ) + + self.assertEqual(signal_only, executor.SignaledV1(digest, b"", b"", 9, False)) + self.assertEqual(actual_oom, executor.OomKilledV1(digest, b"", b"", 2)) + + def test_cgroup_limits_are_read_back_before_execution(self) -> None: + expected = { + b"memory.max": b"67108864\n", + b"memory.swap.max": b"0\n", + b"memory.oom.group": b"1\n", + b"pids.max": b"1\n", + } + _ReadbackCgroup(expected)._require_applied_limits( + memory_max=64 * 1024 * 1024, + pids_max=1, + ) + + for name in expected: + hostile = dict(expected) + hostile[name] = b"max\n" if name != b"memory.max" else b"67112960\n" + with self.subTest(name=name): + with self.assertRaises(OSError) as caught: + _ReadbackCgroup(hostile)._require_applied_limits( + memory_max=64 * 1024 * 1024, + pids_max=1, + ) + self.assertEqual(caught.exception.errno, errno.EPROTO) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + parent = root / "proof" + observer = parent / "observer" + observer.mkdir(parents=True) + (parent / "pids.max").write_bytes(b"2\n") + (parent / "pids.current").write_bytes(b"1\n") + (observer / "pids.current").write_bytes(b"1\n") + with mock.patch.object( + executor, + "_current_unified_cgroup_v1", + return_value=observer, + ): + executor._CgroupV2V1.probe_observer_task_budget(parent) + for path, hostile in ( + (parent / "pids.max", b"3\n"), + (parent / "pids.current", b"2\n"), + (observer / "pids.current", b"2\n"), + ): + original = path.read_bytes() + path.write_bytes(hostile) + with self.subTest(path=path.name, hostile=hostile): + with self.assertRaises(OSError): + executor._CgroupV2V1.probe_observer_task_budget( + parent + ) + path.write_bytes(original) + + def test_observer_initialization_failure_closes_fds_and_reaps_child(self) -> None: + stdin_read, stdin_write = os.pipe() + stdout_read, stdout_write = os.pipe() + stderr_read, stderr_write = os.pipe() + setup_read, setup_write = os.pipe() + pid = os.fork() + if pid == 0: + os.close(stdin_write) + os.close(stdout_read) + os.close(stderr_read) + os.close(setup_read) + while True: + signal.pause() + + os.close(stdin_read) + os.close(stdout_write) + os.close(stderr_write) + os.close(setup_write) + observed_fds = (stdin_write, stdout_read, stderr_read, setup_read) + backend = executor.NativeLinuxBackendV1() + try: + with mock.patch.object( + executor.selectors, + "DefaultSelector", + side_effect=OSError(errno.EMFILE, "selector unavailable"), + ): + result = backend._observe( + _request(), + hashlib.sha256(_static_elf()).digest(), + pid, + _ObserverCgroup(pid), + 0, + *observed_fds, + ) + self.assertEqual( + result, + executor.ObserverFailureV1(executor.ObserverReasonV1.BACKEND_EXCEPTION), + ) + for descriptor in observed_fds: + with self.subTest(descriptor=descriptor): + with self.assertRaises(OSError) as caught: + os.fstat(descriptor) + self.assertEqual(caught.exception.errno, errno.EBADF) + with self.assertRaises(ChildProcessError): + os.waitpid(pid, os.WNOHANG) + finally: + for descriptor in observed_fds: + try: + os.close(descriptor) + except OSError: + pass + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + os.waitpid(pid, 0) + except ChildProcessError: + pass + + def test_fork_rechecks_single_thread_precondition_after_all_setup(self) -> None: + backend = executor.NativeLinuxBackendV1() + cwd_fd = os.open("/", os.O_RDONLY) + try: + with mock.patch.object( + executor.os, + "fork", + side_effect=AssertionError("fork must not run after failed final gate"), + ): + result = backend._fork_and_observe( + _request(cwd=b"/"), + executor._SealedExecutableV1( + 41, + len(_static_elf()), + hashlib.sha256(_static_elf()).digest(), + ), + _LateThreadOperations(), + cwd_fd, + object(), + ) + finally: + os.close(cwd_fd) + + self.assertEqual( + result, + executor.SandboxSetupFailedV1( + hashlib.sha256(_static_elf()).digest(), + b"", + b"", + executor.SetupStageV1.OBSERVER_PRECONDITION, + errno.EBUSY, + ), + ) + + def test_controller_timeout_and_output_limit_are_not_child_outcomes(self) -> None: + digest = hashlib.sha256(_static_elf()).digest() + limits = _limits() + + timeout = executor._classify_process_v1( + digest=digest, + stdout=b"", + stderr=b"", + child_status=9, + oom_kill_delta=1, + residual=False, + setup_packet=b"", + terminal=("timeout", None), + limits=limits, + ) + output = executor._classify_process_v1( + digest=digest, + stdout=b"x" * limits.max_stdout_bytes, + stderr=b"", + child_status=9, + oom_kill_delta=1, + residual=False, + setup_packet=b"", + terminal=("output", executor.OutputStreamV1.STDOUT), + limits=limits, + ) + + self.assertEqual( + timeout, + executor.TimedOutV1(digest, b"", b"", limits.wall_timeout_ns), + ) + self.assertEqual( + output, + executor.OutputLimitExceededV1( + digest, + b"x" * limits.max_stdout_bytes, + b"", + executor.OutputStreamV1.STDOUT, + limits.max_stdout_bytes, + ), + ) + + +@unittest.skipUnless( + sys.platform == "linux" and os.environ.get("LABCOLORS_EXECUTOR_CGROUP_V1"), + "requires Linux and an explicit delegated cgroup v2 parent", +) +class NativeLinuxIntegrationTests(unittest.TestCase): + def setUp(self) -> None: + raw_parent = os.environ["LABCOLORS_EXECUTOR_CGROUP_V1"] + self.cgroup_parent = Path(raw_parent) + self.backend = executor.NativeLinuxBackendV1(self.cgroup_parent) + self.controller = executor.ControlledExecutorV1(self.backend) + report = self.controller.probe() + self.assertEqual( + report, + executor.SupportedV1( + "linux-x86_64", + executor.SANDBOX_POLICY_RELEASE_V1, + ), + ) + + def _native_request( + self, + code: bytes, + *, + stdin: bytes = b"", + stdout_limit: int = 4, + timeout_ns: int = 500_000_000, + memory_max: int = 64 * 1024 * 1024, + ) -> executor.ExecutionRequestV1: + return executor.ExecutionRequestV1( + executable=_linux_executable_elf(code), + argv=(b"native-executor-fixture",), + environment=(), + cwd=b"/", + stdin=stdin, + umask=0o077, + limits=executor.ExecutionLimitsV1( + max_executable_bytes=4096, + max_stdin_bytes=16, + max_argument_bytes=512, + max_stdout_bytes=stdout_limit, + max_stderr_bytes=4, + wall_timeout_ns=timeout_ns, + memory_max_bytes=memory_max, + pids_max=1, + ), + ) + + def _owned_cgroups(self) -> set[str]: + prefix = f"labcolors-executor-{os.getpid()}-" + return { + child.name + for child in self.cgroup_parent.iterdir() + if child.name.startswith(prefix) + } + + def test_real_kernel_success_output_timeout_signal_oom_and_cleanup(self) -> None: + before = self._owned_cgroups() + controlled = self.controller + + exit_request = self._native_request(_LINUX_EXIT_ZERO) + exit_result = controlled.execute(exit_request) + self.assertEqual( + exit_result, + executor.CompletedV1( + hashlib.sha256(exit_request.executable).digest(), + b"", + b"", + ), + ) + + echo_request = self._native_request(_LINUX_ECHO_FOUR, stdin=b"PING") + echo_result = controlled.execute(echo_request) + self.assertEqual( + echo_result, + executor.CompletedV1( + hashlib.sha256(echo_request.executable).digest(), + b"PING", + b"", + ), + ) + + output_request = self._native_request(_LINUX_WRITE_FIVE_AND_LOOP) + output_result = controlled.execute(output_request) + self.assertEqual( + output_result, + executor.OutputLimitExceededV1( + hashlib.sha256(output_request.executable).digest(), + b"1234", + b"", + executor.OutputStreamV1.STDOUT, + 4, + ), + ) + + timeout_request = self._native_request( + _LINUX_BUSY_LOOP, + timeout_ns=50_000_000, + ) + timeout_result = controlled.execute(timeout_request) + self.assertEqual( + timeout_result, + executor.TimedOutV1( + hashlib.sha256(timeout_request.executable).digest(), + b"", + b"", + timeout_request.limits.wall_timeout_ns, + ), + ) + + signal_request = self._native_request(_LINUX_SIGILL) + signal_result = controlled.execute(signal_request) + self.assertEqual( + signal_result, + executor.SignaledV1( + hashlib.sha256(signal_request.executable).digest(), + b"", + b"", + signal.SIGILL, + False, + ), + ) + + foreign_prlimit_request = self._native_request(_LINUX_FOREIGN_PRLIMIT) + foreign_prlimit_result = controlled.execute(foreign_prlimit_request) + self.assertIs(type(foreign_prlimit_result), executor.SignaledV1) + self.assertEqual(foreign_prlimit_result.signal_number, signal.SIGSYS) + + oom_request = self._native_request( + _LINUX_ALLOCATE_UNTIL_OOM, + timeout_ns=5_000_000_000, + memory_max=16 * 1024 * 1024, + ) + oom_result = controlled.execute(oom_request) + self.assertIs(type(oom_result), executor.OomKilledV1) + self.assertGreater(oom_result.oom_kill_delta, 0) + + self.assertEqual(self._owned_cgroups(), before) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/proof/region/v1/arb/tests/test_origin.py b/proof/region/v1/arb/tests/test_origin.py new file mode 100644 index 00000000..a6b43a05 --- /dev/null +++ b/proof/region/v1/arb/tests/test_origin.py @@ -0,0 +1,590 @@ +#!/usr/bin/env python3 +"""Hostile tests for source-origin observations.""" + +from __future__ import annotations + +import hashlib +import gzip +import io +import signal +import sys +import tarfile +import tempfile +import unittest +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + + +PROOF = Path(__file__).resolve().parents[2] +ARB = PROOF / "arb" +sys.path.insert(0, str(PROOF)) +sys.path.insert(0, str(ARB)) + +import origin # noqa: E402 +import provenance # noqa: E402 + + +def git_content_relation_fixture() -> tuple[ + provenance.SourceReleaseLockV1, + provenance.SafeSourceArchiveV1, + origin.GitTreeProcessObservationV1, +]: + common_body = b"license" + omitted_body = b"ci" + generated_body = b"config" + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode="w", format=tarfile.USTAR_FORMAT) as archive: + root = tarfile.TarInfo("fixture-1/") + root.type = tarfile.DIRTYPE + root.mode = 0o755 + root.mtime = 0 + archive.addfile(root) + for name, body, mode in ( + ("fixture-1/LICENSE", common_body, 0o644), + ("fixture-1/configure", generated_body, 0o755), + ): + member = tarfile.TarInfo(name) + member.mode = mode + member.size = len(body) + member.mtime = 0 + archive.addfile(member, io.BytesIO(body)) + archive_bytes = gzip.compress(raw.getvalue(), compresslevel=9, mtime=0) + lock = provenance.SourceReleaseLockV1( + provenance.SourceRoleV1.FLINT_ARB, + "1", + "https://example.invalid/fixture-1.tar.gz", + provenance.ArchiveFormatV1.TAR_GZIP, + len(archive_bytes), + hashlib.sha256(archive_bytes).digest(), + len(raw.getvalue()), + "fixture-1/", + 2, + len(common_body) + len(generated_body), + ( + provenance.LegalFileV1( + "LICENSE", len(common_body), hashlib.sha256(common_body).digest() + ), + ), + provenance.GitContentRelationPolicyV1( + "https://example.invalid/fixture.git", + "v1", + bytes.fromhex("11" * 20), + bytes.fromhex("22" * 20), + 1, + (".github/ci.yml",), + ( + provenance.ProjectPinnedReleaseOnlyFileV1( + "configure", + 0o755, + len(generated_body), + hashlib.sha256(generated_body).digest(), + ), + ), + ), + ) + process = origin.GitTreeProcessObservationV1( + lock.integrity.commit, + lock.integrity.tree, + bytes.fromhex("55" * 32), + ( + origin.FileCoordinateV1( + ".github/ci.yml", 0o644, len(omitted_body), hashlib.sha256(omitted_body).digest() + ), + origin.FileCoordinateV1( + "LICENSE", 0o644, len(common_body), hashlib.sha256(common_body).digest() + ), + ), + bytes.fromhex("33" * 32), + bytes.fromhex("44" * 32), + _token=origin._GIT_PROCESS_TOKEN, + ) + return lock, provenance.admit_source_archive(lock, archive_bytes), process + + +def signed_source_fixture() -> tuple[ + provenance.SourceReleaseLockV1, + provenance.SafeSourceArchiveV1, +]: + base, admitted, _process = git_content_relation_fixture() + packets = origin.decode_public_key_armour((ARB / "keys/gmp.asc").read_bytes()) + signed = replace( + base, + role=provenance.SourceRoleV1.GMP, + integrity=provenance.DetachedSignaturePolicyV1( + "https://example.invalid/fixture-1.tar.gz.sig", + len(b"signature"), + hashlib.sha256(b"signature").digest(), + hashlib.sha256(packets).digest(), + bytes.fromhex("343c2ff0fbee5ec2edbef399f3599ff828c67298"), + ), + ) + return signed, provenance.admit_source_archive(signed, admitted.archive_bytes) + + +class PublicKeyArmourTests(unittest.TestCase): + def test_pinned_key_armour_decodes_to_exact_openpgp_packets(self) -> None: + cases = ( + ( + "gmp.asc", + "928ac84aa0e2134bbb335cd439110dc3f9b967eb04caff4a44dd5d04a3f13474", + ), + ( + "mpfr.asc", + "3fe00f68bbf3888ae185b950d4db0f708dd01b6159cb03dec77296f9045b6372", + ), + ) + for name, expected in cases: + with self.subTest(name=name): + packets = origin.decode_public_key_armour((ARB / "keys" / name).read_bytes()) + self.assertEqual(hashlib.sha256(packets).hexdigest(), expected) + + def test_armour_is_canonical_and_crc_checked(self) -> None: + valid = (ARB / "keys" / "mpfr.asc").read_bytes() + mutants = ( + valid + b"\n", + valid.replace(b"=3az7", b"=3az8", 1), + valid.replace(b"PUBLIC KEY", b"PRIVATE KEY", 1), + valid.replace(b"\n\n", b"\nComment: ambient\n\n", 1), + valid.replace(b"\n", b"\r\n", 1), + ) + for mutant in mutants: + with self.subTest(mutant=hashlib.sha256(mutant).hexdigest()): + with self.assertRaises(origin.OriginErrorV1): + origin.decode_public_key_armour(mutant) + + +class _DiagnosticRunner: + def __init__( + self, + *observations: origin.DiagnosticProcessObservationV1 | object, + ) -> None: + self.observations = list(observations) + self.requests: list[origin.DiagnosticProcessRequestV1] = [] + + def run( + self, + request: origin.DiagnosticProcessRequestV1, + ) -> origin.DiagnosticProcessObservationV1: + self.requests.append(request) + if not self.observations: + raise RuntimeError("unexpected diagnostic invocation") + return self.observations.pop(0) # type: ignore[return-value] + + +class DiagnosticProcessBoundaryTests(unittest.TestCase): + def test_core_has_no_builtin_process_or_process_group_runner(self) -> None: + self.assertFalse(hasattr(origin, "_run_bounded")) + self.assertFalse(hasattr(origin, "subprocess")) + self.assertFalse(hasattr(origin, "selectors")) + + def test_client_owned_diagnostic_bytes_remain_bounded_and_untrusted(self) -> None: + request = origin.DiagnosticProcessRequestV1( + ("verifier", "--version"), + None, + Path("/"), + {"LANG": "C"}, + (), + 1, + 4, + 4, + ) + cases = ( + ( + _DiagnosticRunner( + origin.DiagnosticProcessObservationV1(0, b"12345", b"") + ), + origin.OriginReasonV1.VERIFIER_OUTPUT_LIMIT, + ), + ( + _DiagnosticRunner(SimpleNamespace(returncode=0, stdout=b"", stderr=b"")), + origin.OriginReasonV1.VERIFIER_UNAVAILABLE, + ), + ) + for runner, reason in cases: + with self.subTest(reason=reason): + with self.assertRaises(origin.OriginErrorV1) as caught: + origin._observe_diagnostic_process_v1(runner, request) + self.assertEqual(caught.exception.reason, reason) + + def test_diagnostic_runner_receives_every_resource_bound_explicitly(self) -> None: + observed = origin.DiagnosticProcessObservationV1(0, b"ok", b"") + runner = _DiagnosticRunner(observed) + request = origin.DiagnosticProcessRequestV1( + ("verifier", "arg"), + b"input", + Path("/tmp"), + {"LANG": "C", "TZ": "UTC"}, + (7,), + 3, + 8, + 9, + ) + + actual = origin._observe_diagnostic_process_v1(runner, request) + + self.assertIs(actual, observed) + self.assertEqual(runner.requests, [request]) +class GpgStatusTests(unittest.TestCase): + FINGERPRINT = bytes.fromhex("343c2ff0fbee5ec2edbef399f3599ff828c67298") + + def test_historical_signature_status_is_accepted_despite_later_key_expiry(self) -> None: + status = b"""[GNUPG:] NEWSIG +[GNUPG:] KEYEXPIRED 1736961163 +[GNUPG:] KEY_CONSIDERED 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 0 +[GNUPG:] EXPKEYSIG F3599FF828C67298 Niels Moller +[GNUPG:] VALIDSIG 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 2023-07-30 1690719513 0 4 0 1 10 00 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 +""" + observed = origin.parse_gpgv_status(status, self.FINGERPRINT) + + self.assertIs(type(observed), origin.AcceptedHistoricalSignatureStatusV1) + self.assertEqual(observed.signer_fingerprint, self.FINGERPRINT) + self.assertEqual(observed.signature_unix_time, 1_690_719_513) + + def test_failure_wrong_signer_or_multiple_signatures_are_rejected(self) -> None: + valid = b"""[GNUPG:] NEWSIG +[GNUPG:] VALIDSIG 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 2023-07-30 1690719513 0 4 0 1 10 00 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 +""" + cases = ( + valid.replace(self.FINGERPRINT.hex().upper().encode(), b"A" * 40), + valid.replace(b"2023-07-30", b"2023-07-31", 1), + valid + valid, + valid.replace(b"VALIDSIG", b"BADSIG ", 1), + valid + b"[GNUPG:] FAILURE verify 17\n", + valid + b"unframed stdout\n", + ) + for status in cases: + with self.subTest(status=hashlib.sha256(status).hexdigest()): + with self.assertRaises(origin.OriginErrorV1): + origin.parse_gpgv_status(status, self.FINGERPRINT) + + def test_unbounded_timestamp_and_zero_fingerprint_fail_as_typed_input(self) -> None: + valid = b"""[GNUPG:] NEWSIG +[GNUPG:] VALIDSIG 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 2023-07-30 1690719513 0 4 0 1 10 00 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 +""" + cases = ( + (valid.replace(b"1690719513", b"9" * 400), self.FINGERPRINT), + (valid.replace(self.FINGERPRINT.hex().upper().encode(), b"0" * 40), bytes(20)), + ) + for status, fingerprint in cases: + with self.subTest(status=hashlib.sha256(status).hexdigest()): + with self.assertRaises(origin.OriginErrorV1): + origin.parse_gpgv_status(status, fingerprint) + + def test_signature_observation_is_explicitly_historical_and_diagnostic(self) -> None: + status = b"""[GNUPG:] NEWSIG +[GNUPG:] VALIDSIG 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 2023-07-30 1690719513 0 4 0 1 10 00 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 +""" + signature = b"signature" + expected, admitted = signed_source_fixture() + process = origin.GpgvProcessObservationV1( + 0, + status, + b"", + admitted.tree_identity, + admitted.archive_sha256, + hashlib.sha256(signature).digest(), + expected.integrity.public_key_packets_sha256, + bytes.fromhex("11" * 32), + bytes.fromhex("22" * 32), + _token=origin._GPGV_PROCESS_TOKEN, + ) + observed = origin.admit_detached_signature_observation( + expected=expected, + admitted=admitted, + signature=signature, + public_key_armour=(ARB / "keys/gmp.asc").read_bytes(), + process=process, + ) + + self.assertIs( + type(observed), + origin.HistoricalPathRecheckedSignatureDiagnosticV1, + ) + for attribute in ( + "authenticated_source", + "current_publisher", + "currently_trusted", + "publisher", + "verified_publisher", + ): + with self.subTest(attribute=attribute): + self.assertFalse(hasattr(observed, attribute)) + + def test_process_observation_cannot_report_other_source_bytes(self) -> None: + status = b"""[GNUPG:] NEWSIG +[GNUPG:] VALIDSIG 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 2023-07-30 1690719513 0 4 0 1 10 00 343C2FF0FBEE5EC2EDBEF399F3599FF828C67298 +""" + expected, admitted = signed_source_fixture() + process = origin.GpgvProcessObservationV1( + 0, + status, + b"", + bytes.fromhex("ff" * 32), + admitted.archive_sha256, + hashlib.sha256(b"signature").digest(), + expected.integrity.public_key_packets_sha256, + bytes.fromhex("11" * 32), + bytes.fromhex("22" * 32), + _token=origin._GPGV_PROCESS_TOKEN, + ) + with self.assertRaises(origin.OriginErrorV1) as caught: + origin.admit_detached_signature_observation( + expected=expected, + admitted=admitted, + signature=b"signature", + public_key_armour=(ARB / "keys/gmp.asc").read_bytes(), + process=process, + ) + self.assertEqual(caught.exception.reason, origin.OriginReasonV1.COORDINATE_MISMATCH) + + def test_crashed_gpgv_is_a_typed_process_failure(self) -> None: + _expected, admitted = signed_source_fixture() + with tempfile.TemporaryDirectory() as temporary: + executable = Path(temporary) / "gpgv" + executable.write_bytes(b"diagnostic executable bytes") + runner = _DiagnosticRunner( + origin.DiagnosticProcessObservationV1(0, b"gpgv fixture\n", b""), + origin.DiagnosticProcessObservationV1(-signal.SIGSEGV, b"", b""), + ) + + with self.assertRaises(origin.OriginErrorV1) as caught: + origin.run_gpgv( + admitted, + b"signature", + (ARB / "keys/gmp.asc").read_bytes(), + executable=executable, + runner=runner, + ) + self.assertEqual(caught.exception.reason, origin.OriginReasonV1.VERIFIER_FAILED) + + def test_process_and_signature_diagnostic_have_no_public_constructor(self) -> None: + with self.assertRaises(TypeError): + origin.GpgvProcessObservationV1( + 0, + b"[GNUPG:] NEWSIG\n", + b"", + bytes.fromhex("88" * 32), + bytes.fromhex("99" * 32), + bytes.fromhex("aa" * 32), + bytes.fromhex("bb" * 32), + bytes.fromhex("11" * 32), + bytes.fromhex("22" * 32), + _token=object(), + ) + with self.assertRaises(TypeError): + origin.admit_detached_signature_observation( + expected=signed_source_fixture()[0], + admitted=signed_source_fixture()[1], + signature=b"fake", + public_key_armour=(ARB / "keys/gmp.asc").read_bytes(), + process=SimpleNamespace(returncode=0, status=b"self report"), + ) + with self.assertRaises(TypeError): + origin.HistoricalPathRecheckedSignatureDiagnosticV1( + bytes.fromhex("11" * 32), + bytes.fromhex("22" * 32), + bytes.fromhex("33" * 32), + bytes.fromhex("77" * 32), + self.FINGERPRINT, + 1, + bytes.fromhex("44" * 32), + bytes.fromhex("55" * 32), + _token=object(), + ) + + def test_old_exact_or_current_authority_symbols_do_not_exist(self) -> None: + for name in ( + "ExactGpgvSignatureObservationV1", + "PathRecheckedSignatureObservationV1", + "ValidSignatureObservationV1", + ): + with self.subTest(name=name): + self.assertFalse(hasattr(origin, name)) + self.assertNotIn( + "same_object_exec", + origin.GpgvProcessObservationV1.__dataclass_fields__, + ) + + +class GitRelationTests(unittest.TestCase): + def test_git_batch_recomputes_blob_object_identity(self) -> None: + body = b"value" + object_id = hashlib.sha1(b"blob 5\0" + body).hexdigest().encode("ascii") + listing = ((object_id, "value", 0o644),) + valid = object_id + b" blob 5\n" + body + b"\n" + + self.assertEqual(origin._parse_git_batch(valid, listing)[0].sha256, hashlib.sha256(body).digest()) + with self.assertRaises(origin.OriginErrorV1): + origin._parse_git_batch(b"2" * 40 + valid[40:], ((b"2" * 40, "value", 0o644),)) + + def test_recursive_tree_identity_has_an_independent_git_golden(self) -> None: + listing = ( + (b"8c7e5a667f1b771847fe88c01c3de34413a1b220", "a.c", 0o644), + (b"7371f47a6f8bd23a8fa1a8b2a9479cdd76380e54", "dir/b", 0o644), + ) + self.assertEqual( + origin._recompute_git_tree_identity(listing).hex(), + "3930f0d390a7a4f2b29fde1dbc8abdc98a282fe0", + ) + + def test_deep_valid_git_tree_is_iterative_not_a_python_stack_overflow(self) -> None: + body = b"z" + object_id = hashlib.sha1(b"blob 1\0" + body).hexdigest().encode("ascii") + path = "/".join(("a",) * 1_500 + ("z",)) + listing = origin._parse_git_listing( + b"100644 blob " + object_id + b"\t" + path.encode("ascii") + b"\0" + ) + + tree = origin._recompute_git_tree_identity(listing) + + self.assertEqual(len(tree), 20) + self.assertNotEqual(tree, bytes(20)) + + def test_malformed_git_output_never_escapes_the_typed_boundary(self) -> None: + for raw in ( + b"100644 blob " + b"0" * 40 + b"\tvalue\0", + b"100644 blob " + b"1" * 40 + b"\t../escape\0", + b"100644 blob " + b"1" * 40 + b"\ta\nb\0", + ): + with self.subTest(raw=raw): + with self.assertRaises(origin.OriginErrorV1): + origin._parse_git_listing(raw) + + def test_git_batch_length_is_canonical_decimal(self) -> None: + body = b"value" + object_id = hashlib.sha1(b"blob 5\0" + body).hexdigest().encode("ascii") + listing = ((object_id, "value", 0o644),) + for length in (b"+5", b"05", b" 5"): + with self.subTest(length=length): + raw = object_id + b" blob " + length + b"\n" + body + b"\n" + with self.assertRaises(origin.OriginErrorV1): + origin._parse_git_batch(raw, listing) + + def test_commit_identity_and_commit_to_tree_edge_are_recomputed(self) -> None: + tree = bytes.fromhex("22" * 20) + body = b"tree " + tree.hex().encode("ascii") + b"\nauthor A 0 +0000\ncommitter A 0 +0000\n\nrelease\n" + commit = hashlib.sha1(b"commit " + str(len(body)).encode("ascii") + b"\0" + body).digest() + + self.assertEqual(origin._admit_git_commit_object(body, commit, tree), hashlib.sha256(body).digest()) + for changed_body, changed_commit, changed_tree in ( + (body + b"x", commit, tree), + (body, bytes.fromhex("ff" * 20), tree), + (body, commit, bytes.fromhex("ff" * 20)), + ): + with self.assertRaises(origin.OriginErrorV1): + origin._admit_git_commit_object(changed_body, changed_commit, changed_tree) + + def test_archive_is_common_tree_plus_project_pinned_release_only_files(self) -> None: + lock, admitted, process = git_content_relation_fixture() + + evidence = origin.admit_git_content_relation_observation( + expected=lock, + admitted=admitted, + process=process, + ) + + self.assertEqual(evidence.common_file_count, 1) + self.assertEqual(evidence.omitted_file_count, 1) + self.assertEqual(evidence.project_pinned_release_only_file_count, 1) + self.assertEqual(evidence.archive_sha256, lock.archive_sha256) + self.assertIs(type(evidence), origin.RecomputedGitContentRelationV1) + + def test_any_relation_edge_mismatch_is_rejected(self) -> None: + lock, admitted, process = git_content_relation_fixture() + base = dict(expected=lock, admitted=admitted, process=process) + mutations = ( + {"expected": replace(lock, version="2")}, + { + "expected": replace( + lock, + integrity=replace( + lock.integrity, + commit=bytes.fromhex("ff" * 20), + ), + ) + }, + {"process": SimpleNamespace(**process.__dict__)}, + ) + for mutation in mutations: + with self.subTest(mutation=mutation): + with self.assertRaises((origin.OriginErrorV1, TypeError)): + origin.admit_git_content_relation_observation(**(base | mutation)) + + def test_git_executable_metadata_has_no_relation_authority(self) -> None: + lock, admitted, process = git_content_relation_fixture() + other_verifier = origin.GitTreeProcessObservationV1( + process.commit, + process.tree, + process.commit_object_sha256, + process.files, + bytes.fromhex("aa" * 32), + bytes.fromhex("bb" * 32), + _token=origin._GIT_PROCESS_TOKEN, + ) + + first = origin.admit_git_content_relation_observation( + expected=lock, admitted=admitted, process=process + ) + second = origin.admit_git_content_relation_observation( + expected=lock, admitted=admitted, process=other_verifier + ) + + self.assertEqual(first, second) + self.assertFalse(hasattr(first, "verifier_executable_sha256")) + self.assertFalse(hasattr(first, "verifier_version_sha256")) + + def test_control_bytes_are_not_source_coordinates(self) -> None: + for path in ("a\0b", "a\nb", "a\x7fb"): + with self.subTest(path=repr(path)): + with self.assertRaises(TypeError): + origin.FileCoordinateV1( + path, + 0o644, + 1, + hashlib.sha256(b"a").digest(), + ) + + def test_process_and_verified_types_have_no_public_constructor(self) -> None: + with self.assertRaises(TypeError): + origin.GitTreeProcessObservationV1( + bytes.fromhex("11" * 20), + bytes.fromhex("22" * 20), + bytes.fromhex("55" * 32), + (origin.FileCoordinateV1("a", 0o644, 1, hashlib.sha256(b"a").digest()),), + bytes.fromhex("33" * 32), + bytes.fromhex("44" * 32), + _token=object(), + ) + with self.assertRaises(TypeError): + origin.RecomputedGitContentRelationV1( + bytes.fromhex("11" * 32), + bytes.fromhex("22" * 32), + bytes.fromhex("33" * 20), + bytes.fromhex("44" * 20), + bytes.fromhex("55" * 32), + bytes.fromhex("66" * 32), + bytes.fromhex("77" * 32), + 1, + 1, + 1, + _token=object(), + ) + + def test_old_git_authority_symbols_do_not_exist(self) -> None: + for name in ( + "ExactGitRelationObservationV1", + "PathRecheckedGitRelationObservationV1", + "admit_git_release_observation", + ): + with self.subTest(name=name): + self.assertFalse(hasattr(origin, name)) + self.assertNotIn( + "same_object_exec", + origin.GitTreeProcessObservationV1.__dataclass_fields__, + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/proof/region/v1/arb/tests/test_pipeline.py b/proof/region/v1/arb/tests/test_pipeline.py new file mode 100644 index 00000000..9ea6baf8 --- /dev/null +++ b/proof/region/v1/arb/tests/test_pipeline.py @@ -0,0 +1,1458 @@ +#!/usr/bin/env python3 +"""Causal, hostile tests for the controlled Arb BUILD/RUN pipeline.""" + +from __future__ import annotations + +import gzip +import hashlib +import io +import os +import stat +import struct +import subprocess +import sys +import tarfile +import tempfile +import unittest +from dataclasses import fields as dataclass_fields, replace +from functools import cache +from pathlib import Path +from types import MethodType +from unittest import mock + + +PROOF = Path(__file__).resolve().parents[2] +ARB = PROOF / "arb" +REPO = PROOF.parents[2] +sys.path.insert(0, str(PROOF)) +sys.path.insert(0, str(ARB)) + +import executor # noqa: E402 +import pipeline # noqa: E402 +import provenance # noqa: E402 +from region_proof_protocol import ( # noqa: E402 + ComparatorKindV1, + ComparatorManifestV2, + ContentResolvedComparatorManifestV2, + DecisionTranscriptV1, + DecisionV1, + ProofJobV1, + ProtocolErrorV1, +) + + +def _digest(label: str) -> bytes: + return hashlib.sha256(label.encode("ascii")).digest() + + +def _static_elf(payload: bytes = b"fixture") -> bytes: + """Return a parseable static ELF64/x86-64 object for executor admission.""" + + code_offset = 64 + 56 + body = payload or b"x" + file_size = code_offset + len(body) + ident = b"\x7fELF\x02\x01\x01" + bytes(9) + header = ident + struct.pack( + " tuple[bytes, int]: + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode="w", format=tarfile.USTAR_FORMAT) as archive: + directories = {root} + for relative, _body, _mode in files: + parent = Path(relative).parent + while str(parent) not in ("", "."): + directories.add(f"{root}/{parent.as_posix()}") + parent = parent.parent + for name in sorted(directories, key=lambda item: (item.count("/"), item)): + member = tarfile.TarInfo(f"{name}/") + member.type = tarfile.DIRTYPE + member.mode = 0o755 + member.mtime = 0 + archive.addfile(member) + for relative, body, mode in files: + member = tarfile.TarInfo(f"{root}/{relative}") + member.mode = mode + member.size = len(body) + member.mtime = 0 + archive.addfile(member, io.BytesIO(body)) + encoded = gzip.compress(raw.getvalue(), compresslevel=9, mtime=0) + return encoded, len(raw.getvalue()) + + +@cache +def _source_fixture() -> tuple[ + provenance.ArbSourceLockV1, + provenance.AdmittedArbSourcesV1, +]: + locks: list[provenance.SourceReleaseLockV1] = [] + safe: list[provenance.SafeSourceArchiveV1] = [] + coordinates = ( + (provenance.SourceRoleV1.GMP, "gmp-6.3.0", False), + (provenance.SourceRoleV1.MPFR, "mpfr-4.2.2", False), + (provenance.SourceRoleV1.FLINT_ARB, "flint-3.6.0", True), + ) + for index, (role, root, git) in enumerate(coordinates, start=1): + files = (("LICENSE", f"license-{index}".encode(), 0o644),) + if git: + files += (("configure", b"generated", 0o755),) + archive, raw_length = _tar(root, files) + if git: + integrity: provenance.SourceIntegrityPolicyV1 = provenance.GitContentRelationPolicyV1( + "https://example.invalid/flint.git", + "v1", + bytes.fromhex("11" * 20), + bytes.fromhex("22" * 20), + 1, + ("ci/omitted",), + ( + provenance.ProjectPinnedReleaseOnlyFileV1( + "configure", + 0o755, + len(b"generated"), + hashlib.sha256(b"generated").digest(), + ), + ), + ) + else: + integrity = provenance.DetachedSignaturePolicyV1( + f"https://example.invalid/{root}.tar.gz.sig", + 3, + _digest(f"signature-{index}"), + _digest(f"public-key-{index}"), + bytes((index,)) * 20, + ) + lock = provenance.SourceReleaseLockV1( + role, + "1", + f"https://example.invalid/{root}.tar.gz", + provenance.ArchiveFormatV1.TAR_GZIP, + len(archive), + hashlib.sha256(archive).digest(), + raw_length, + f"{root}/", + len(files), + sum(len(body) for _name, body, _mode in files), + ( + provenance.LegalFileV1( + "LICENSE", + len(files[0][1]), + hashlib.sha256(files[0][1]).digest(), + ), + ), + integrity, + ) + locks.append(lock) + safe.append(provenance.admit_source_archive(lock, archive)) + source_lock = provenance.ArbSourceLockV1(tuple(locks)) + admitted = provenance.admit_arb_sources(source_lock, tuple(safe)) + return source_lock, admitted + + +@cache +def _generated_formula() -> bytes: + result = subprocess.run( + ( + sys.executable, + str(ARB / "evaluator/formula.py"), + str(REPO / "crates/labcolors-core/contracts/contextual-region-formula-v1.lcir"), + ), + check=False, + capture_output=True, + env={"PYTHONDONTWRITEBYTECODE": "1", "PYTHONHASHSEED": "0"}, + timeout=30, + ) + if result.returncode != 0: + raise AssertionError(result.stderr.decode("utf-8", "replace")) + return result.stdout + + +@cache +def _build_sources() -> pipeline.AdmittedBuildSourcesV1: + files = [] + for logical_path, mode in pipeline.REQUIRED_BUILD_SOURCE_MODES_V1: + if logical_path == pipeline.GENERATED_FORMULA_PATH_V1: + body = _generated_formula() + else: + body = (REPO / logical_path).read_bytes() + files.append(pipeline.BuildSourceFileV1(logical_path, mode, body)) + return pipeline.admit_build_sources_v1(tuple(files)) + + +@cache +def _job() -> ProofJobV1: + return ProofJobV1.parse((PROOF / "fixtures/proof-job-v1.bin").read_bytes()) + + +@cache +def _foreign_comparator() -> ContentResolvedComparatorManifestV2: + content = tuple(f"manifest-coordinate-{index}".encode() for index in range(10)) + manifest = ComparatorManifestV2( + ComparatorKindV1.ARB, + *(hashlib.sha256(item).digest() for item in content), + ) + by_digest = {hashlib.sha256(item).digest(): item for item in content} + return ContentResolvedComparatorManifestV2.admit(manifest, by_digest.get) + + +@cache +def _transcript( + manifest_identity: bytes = _digest("foreign-manifest-identity"), +) -> bytes: + job = _job() + transcript = DecisionTranscriptV1.from_decisions( + job, + _foreign_comparator(), + (DecisionV1.OUTSIDE for _ in range(job.domain.point_count)), + (), + _digest("accounting"), + ) + return replace(transcript, comparator_identity=manifest_identity).encode() + + +def _limits() -> executor.ExecutionLimitsV1: + return executor.ExecutionLimitsV1( + max_executable_bytes=16 * 1024 * 1024, + max_stdin_bytes=16 * 1024 * 1024, + max_argument_bytes=4096, + max_stdout_bytes=16 * 1024 * 1024, + max_stderr_bytes=64 * 1024, + wall_timeout_ns=60_000_000_000, + memory_max_bytes=1024 * 1024 * 1024, + pids_max=1, + ) + + +def _request(**changes: object) -> pipeline.PipelineRequestV1: + source_lock, admitted = _source_fixture() + values: dict[str, object] = { + "source_lock": source_lock, + "admitted_sources": admitted, + "build_sources": _build_sources(), + "job": _job(), + "execution_limits": _limits(), + "host_trust": pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST, + } + values.update(changes) + return pipeline.PipelineRequestV1(**values) + + +class _BuildBackend: + def __init__( + self, + outputs: tuple[bytes, ...], + *, + probe: pipeline.DockerCapabilityReportV1 | None = None, + mutate_inputs: bool = False, + hardlink_input: bool = False, + symlink_output: bool = False, + reported_stdout: bytes | None = None, + ) -> None: + self.outputs = list(outputs) + self.probe_result = probe or pipeline.DockerSupportedV1( + pipeline.OCI_IMAGE_REFERENCE_V1, + pipeline.OCI_PLATFORM_V1, + _digest("docker-daemon"), + ) + self.mutate_inputs = mutate_inputs + self.hardlink_input = hardlink_input + self.symlink_output = symlink_output + self.reported_stdout = reported_stdout + self.requests: list[pipeline.DockerBuildRequestV1] = [] + + def probe(self) -> pipeline.DockerCapabilityReportV1: + return self.probe_result + + def run_build( + self, + request: pipeline.DockerBuildRequestV1, + ) -> pipeline.DockerBuildProcessObservationV1: + self.requests.append(request) + output = self.outputs.pop(0) + target = request.output_directory / pipeline.EVALUATOR_OUTPUT_NAME_V1 + if self.symlink_output: + outside = request.root_directory / "outside" + outside.write_bytes(output) + target.symlink_to(outside) + else: + target.write_bytes(output) + target.chmod(0o555) + if self.mutate_inputs: + victim = request.workspace_directory / "proof/region/v1/arb/evaluator/main.c" + victim.chmod(0o644) + victim.write_bytes(b"mutated") + if self.hardlink_input: + victim = request.workspace_directory / "proof/region/v1/arb/evaluator/main.c" + outside = request.root_directory / "input-hardlink" + outside.write_bytes(victim.read_bytes()) + outside.chmod(0o644) + victim.unlink() + os.link(outside, victim) + return pipeline.DockerBuildExitedV1( + 0, + self.reported_stdout + if self.reported_stdout is not None + else b"sha256:" + _digest("self-reported-output").hex().encode(), + b"", + ) + + +class _Executor: + def __init__(self, result_factory: object | None = None) -> None: + self.requests: list[executor.ExecutionRequestV1] = [] + self.capabilities: list[executor.SupportedV1] = [] + self.results: list[executor.ExecutionResultV1] = [] + self.result_factory = result_factory + self.probe_calls = 0 + + def probe(self) -> executor.CapabilityReportV1: + self.probe_calls += 1 + return executor.SupportedV1( + "linux-x86_64", + executor.SANDBOX_POLICY_RELEASE_V1, + ) + + def execute( + self, + request: executor.ExecutionRequestV1, + capability: executor.SupportedV1, + ) -> executor.ExecutionResultV1: + self.requests.append(request) + self.capabilities.append(capability) + if self.result_factory is not None: + result = self.result_factory(request) + else: + manifest_identity = bytes.fromhex(request.argv[2].decode("ascii")) + result = executor.CompletedV1( + hashlib.sha256(request.executable).digest(), + _transcript(manifest_identity), + b"", + ) + self.results.append(result) + return result + + +class _MasqueradingControlledExecutor(executor.ControlledExecutorV1): + pass + + +class _MasqueradingNativeBackend(executor.NativeLinuxBackendV1): + def probe(self, _guard: object) -> executor.CapabilityReportV1: + return executor.SupportedV1( + "linux-x86_64", + executor.SANDBOX_POLICY_RELEASE_V1, + ) + + def run( + self, + request: executor.ExecutionRequestV1, + _capability: executor.SupportedV1, + ) -> executor.ExecutionResultV1: + manifest_identity = bytes.fromhex(request.argv[2].decode("ascii")) + return executor.CompletedV1( + hashlib.sha256(request.executable).digest(), + _transcript(manifest_identity), + b"", + ) + + +class _SelfMutatingExecutionBackend: + owner: executor.ControlledExecutorV1 + + def probe(self, _guard: object) -> executor.CapabilityReportV1: + return executor.SupportedV1( + "linux-x86_64", + executor.SANDBOX_POLICY_RELEASE_V1, + ) + + def run( + self, + request: executor.ExecutionRequestV1, + _capability: executor.SupportedV1, + ) -> executor.ExecutionResultV1: + self.owner._backend = executor.NativeLinuxBackendV1( + Path("/sys/fs/cgroup/labcolors") + ) + manifest_identity = bytes.fromhex(request.argv[2].decode("ascii")) + return executor.CompletedV1( + hashlib.sha256(request.executable).digest(), + _transcript(manifest_identity), + b"", + ) + + +class BuildSourceAdmissionTests(unittest.TestCase): + def test_exact_formula_generator_recipe_and_evaluator_bytes_are_admitted(self) -> None: + admitted = _build_sources() + + self.assertEqual(admitted.formula_spec, _job().formula_spec) + self.assertEqual( + hashlib.sha256(admitted.generated_formula).hexdigest(), + pipeline.GENERATED_FORMULA_SHA256_V1, + ) + self.assertEqual( + tuple(item.path for item in admitted.files), + tuple(path for path, _mode in pipeline.REQUIRED_BUILD_SOURCE_MODES_V1), + ) + self.assertNotEqual(admitted.build_input_identity, admitted.formula_support_identity) + self.assertFalse(hasattr(admitted, "source_path")) + + def test_missing_extra_reordered_or_mutated_source_bytes_are_rejected(self) -> None: + files = _build_sources().files + mutants = ( + files[:-1], + files + (pipeline.BuildSourceFileV1("extra.c", 0o644, b"x"),), + tuple(reversed(files)), + (replace(files[0], contents=files[0].contents + b"x"),) + files[1:], + ) + for mutant in mutants: + with self.subTest(length=len(mutant)): + with self.assertRaises(pipeline.BuildSourceAdmissionErrorV1): + pipeline.admit_build_sources_v1(mutant) + + def test_capabilities_cannot_be_directly_forged(self) -> None: + with self.assertRaises(TypeError): + pipeline.AdmittedBuildSourcesV1( + _build_sources().files, + _digest("forged"), + _token=object(), + ) + + +class FlintSourcePartitionTests(unittest.TestCase): + def test_partition_is_nonempty_and_separately_binds_both_content_sets(self) -> None: + source_lock, admitted = _source_fixture() + flint = source_lock.sources[2] + + partition = pipeline.flint_source_content_partition_v1( + source_lock, + admitted, + ) + + self.assertGreater(partition.commit_content_file_count, 0) + self.assertGreater(partition.project_pinned_release_only_file_count, 0) + self.assertEqual( + partition.commit_content_file_count, + flint.integrity.common_file_count, + ) + self.assertEqual( + partition.project_pinned_release_only_file_count, + len(flint.integrity.project_pinned_release_only_files), + ) + self.assertEqual( + partition.commit_content_file_count + + partition.project_pinned_release_only_file_count, + admitted.sources[2].regular_file_count, + ) + self.assertNotEqual( + partition.commit_content_identity, + partition.project_pinned_release_only_identity, + ) + self.assertFalse(hasattr(partition, "commit_derived_identity")) + + def test_partition_rejects_a_foreign_lock_replay(self) -> None: + source_lock, admitted = _source_fixture() + foreign_flint = replace( + source_lock.sources[2], + version="foreign-release", + ) + foreign_lock = provenance.ArbSourceLockV1( + source_lock.sources[:2] + (foreign_flint,) + ) + + with self.assertRaises(pipeline.PipelineInputErrorV1) as caught: + pipeline.flint_source_content_partition_v1(foreign_lock, admitted) + + self.assertEqual( + caught.exception.reason, + pipeline.PipelineInputReasonV1.FOREIGN_SOURCE_CAPABILITY, + ) + + +class ComparatorDerivationTests(unittest.TestCase): + def _result(self) -> pipeline.DiagnosticPipelineObservationV1: + binary = _static_elf(b"derived-comparator") + result = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((binary, binary)), + execution_controller=_Executor(), + ).execute(_request()) + self.assertIs(type(result), pipeline.DiagnosticPipelineObservationV1) + return result + + def test_request_cannot_supply_an_arbitrary_comparator(self) -> None: + with self.assertRaises(TypeError): + _request(comparator=_foreign_comparator()) + + def test_all_ten_coordinates_replay_exact_named_preimages(self) -> None: + result = self._result() + admitted = result.comparator + manifest = admitted.manifest.manifest + + names = tuple(field.name for field in dataclass_fields(admitted.preimages)) + self.assertEqual( + names, + ( + "engine_release", + "upstream_source", + "arithmetic_input_set", + "wrapper_source", + "evaluator_source", + "build_identity", + "operation_allowlist", + "test_observation", + "legal_file_set", + "exclusions", + ), + ) + for name in names: + with self.subTest(name=name): + preimage = getattr(admitted.preimages, name) + self.assertGreater(len(preimage), len(name)) + self.assertNotEqual(preimage, name.encode("ascii")) + self.assertEqual( + getattr(manifest, name), + hashlib.sha256(preimage).digest(), + ) + self.assertEqual(admitted.identity, admitted.manifest.identity) + self.assertEqual( + admitted.structural_source_identity, + result.structural_source_identity, + ) + self.assertEqual(admitted.build_input_identity, result.build_input_identity) + self.assertEqual(admitted.pipeline_policy_identity, result.pipeline_policy_identity) + self.assertEqual(admitted.binary_sha256, result.binary_sha256) + self.assertEqual(admitted.rebuild_sha256s, result.rebuild_sha256s) + self.assertIn( + b"gap:host-and-docker-daemon-not-source-bound", + admitted.preimages.exclusions, + ) + self.assertNotIn(b"persistent", admitted.preimages.exclusions) + self.assertNotIn(b"github-hosted", admitted.preimages.exclusions) + + def test_mutated_or_reordered_preimages_cannot_replay_the_manifest(self) -> None: + admitted = self._result().comparator + manifest = admitted.manifest.manifest + original = { + getattr(manifest, field.name): getattr(admitted.preimages, field.name) + for field in dataclass_fields(admitted.preimages) + } + variants = [] + mutated = dict(original) + mutated[manifest.evaluator_source] += b"x" + variants.append(mutated) + reordered = dict(original) + reordered[manifest.wrapper_source], reordered[manifest.evaluator_source] = ( + reordered[manifest.evaluator_source], + reordered[manifest.wrapper_source], + ) + variants.append(reordered) + + for resolver in variants: + with self.subTest(variant=variants.index(resolver)): + with self.assertRaises(ProtocolErrorV1): + ContentResolvedComparatorManifestV2.admit( + manifest, + resolver.get, + ) + + def test_operator_coordinate_is_the_exact_ordered_formula_contract(self) -> None: + original = _build_sources().formula_spec + lines = original.splitlines() + self.assertIn( + b"operators 20", + lines, + "registered formula must retain the exact 20-operator contract", + ) + count_index = lines.index(b"operators 20") + lines[count_index + 1], lines[count_index + 2] = ( + lines[count_index + 2], + lines[count_index + 1], + ) + reordered = b"\n".join(lines) + b"\n" + + original_preimage = pipeline._operation_allowlist_preimage_v1(original) + reordered_preimage = pipeline._operation_allowlist_preimage_v1(reordered) + + self.assertNotEqual(original_preimage, reordered_preimage) + self.assertNotEqual( + hashlib.sha256(original_preimage).digest(), + hashlib.sha256(reordered_preimage).digest(), + ) + + def test_wrapper_and_evaluator_file_sets_are_exact_and_disjoint(self) -> None: + result = self._result() + files = _build_sources().files + wrapper_paths = frozenset( + ( + "proof/region/v1/arb/evaluator/formula.h", + "proof/region/v1/arb/evaluator/interval.c", + "proof/region/v1/arb/evaluator/interval.h", + ) + ) + excluded = wrapper_paths | { + pipeline.FORMULA_SPEC_PATH_V1, + pipeline.FORMULA_GENERATOR_PATH_V1, + pipeline.BUILD_RECIPE_PATH_V1, + } + wrapper_files = tuple(item for item in files if item.path in wrapper_paths) + evaluator_files = tuple(item for item in files if item.path not in excluded) + + self.assertFalse({item.path for item in wrapper_files} & {item.path for item in evaluator_files}) + self.assertEqual( + result.comparator.preimages.wrapper_source, + pipeline._encoded_build_file_set_v1( + b"labcolors.proof-region.arb-comparator.wrapper-source.v1\0", + wrapper_files, + ), + ) + self.assertEqual( + result.comparator.preimages.evaluator_source, + pipeline._encoded_build_file_set_v1( + b"labcolors.proof-region.arb-comparator.evaluator-source.v1\0", + evaluator_files, + ), + ) + + def test_build_stdout_cannot_supply_a_foreign_manifest_or_coordinate(self) -> None: + foreign = _foreign_comparator() + report = b"manifest=" + foreign.identity.hex().encode("ascii") + binary = _static_elf(b"ignore-build-self-report") + + result = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend( + (binary, binary), + reported_stdout=report, + ), + execution_controller=_Executor(), + ).execute(_request()) + + self.assertIs(type(result), pipeline.DiagnosticPipelineObservationV1) + self.assertNotEqual(result.comparator.identity, foreign.identity) + coordinates = tuple( + getattr(result.comparator.manifest.manifest, field.name) + for field in dataclass_fields(result.comparator.manifest.manifest) + if field.name != "kind" + ) + self.assertNotIn(foreign.identity, coordinates) + self.assertEqual(result.build_processes[0].stdout, report) + + def test_foreign_comparator_transcript_is_rejected(self) -> None: + binary = _static_elf(b"foreign-transcript") + run = _Executor( + lambda request: executor.CompletedV1( + hashlib.sha256(request.executable).digest(), + _transcript(_foreign_comparator().identity), + b"", + ) + ) + + result = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((binary, binary)), + execution_controller=run, + ).execute(_request()) + + self.assertIs(type(result), pipeline.TranscriptRejectedV1) + self.assertEqual(result.reason, pipeline.TranscriptFailureReasonV1.FOREIGN_BINDING) + + def test_diagnostic_comparator_has_no_public_constructor(self) -> None: + with self.assertRaises(TypeError): + pipeline.DiagnosticArbComparatorV1() + + +class ControlledPipelineTests(unittest.TestCase): + def test_admission_uses_only_explicit_cross_module_verification_api(self) -> None: + source = (ARB / "pipeline.py").read_text(encoding="utf-8") + + for forbidden in ( + "executor._result_matches_request", + "executor._require_static_x86_64_elf", + "protocol._validate_witness_alignment", + ): + with self.subTest(forbidden=forbidden): + self.assertNotIn(forbidden, source) + + def test_host_trust_claims_only_backend_observable_facts(self) -> None: + trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST + + self.assertEqual(tuple(pipeline.HostTrustBoundaryV1), (trust,)) + self.assertEqual(trust.value, "unsealed-linux-x64-docker-host") + self.assertFalse( + hasattr( + pipeline.HostTrustBoundaryV1, + "PERSISTENT_SELF_HOSTED_DOCKER", + ) + ) + + def test_pipeline_policy_identity_binds_the_snapshot_timestamp_policy(self) -> None: + trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST + original = pipeline.pipeline_policy_identity_v1(trust) + + with mock.patch.object( + pipeline.snapshot, + "SOURCE_SNAPSHOT_MTIME_NS_V1", + pipeline.snapshot.SOURCE_SNAPSHOT_MTIME_NS_V1 + 1, + ): + changed = pipeline.pipeline_policy_identity_v1(trust) + + self.assertNotEqual(original, changed) + + def test_pipeline_policy_identity_binds_the_private_tmpfs_policy(self) -> None: + trust = pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST + original = pipeline.pipeline_policy_identity_v1(trust) + + with mock.patch.object( + pipeline, + "_BUILD_TMPFS_SPEC_V1", + "/tmp:rw,exec,suid,dev,mode=1777", + ): + changed = pipeline.pipeline_policy_identity_v1(trust) + + self.assertNotEqual(original, changed) + + def test_build_only_does_not_probe_or_execute_run_backend(self) -> None: + binary = _static_elf(b"build-only") + controller = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((binary, binary)), + execution_controller=None, + ) + + result = controller.build(_request()) + + self.assertIs(type(result), pipeline.DiagnosticBuildObservationV1) + self.assertEqual(result.binary, binary) + self.assertEqual(result.rebuild_sha256s, (result.binary_sha256,) * 2) + self.assertIs(type(result.comparator), pipeline.DiagnosticArbComparatorV1) + + def test_two_fresh_equal_builds_feed_exact_observed_bytes_to_executor(self) -> None: + binary = _static_elf(b"observed-output") + build = _BuildBackend((binary, binary)) + run = _Executor() + controller = pipeline.ControlledPipelineV1(build_backend=build, execution_controller=run) + + result = controller.execute(_request()) + + self.assertIs(type(result), pipeline.DiagnosticPipelineObservationV1) + self.assertEqual(len(build.requests), 2) + self.assertEqual(tuple(item.attempt for item in build.requests), (1, 2)) + self.assertNotEqual( + build.requests[0].root_directory, + build.requests[1].root_directory, + ) + self.assertTrue( + all(not item.root_directory.exists() for item in build.requests), + "fresh build roots must be removed after post-exit observation", + ) + self.assertEqual(len(run.requests), 1) + self.assertEqual(run.probe_calls, 1) + self.assertIs(type(run.capabilities[0]), executor.SupportedV1) + self.assertIs(run.requests[0].executable, result.binary) + self.assertEqual(result.binary, binary) + self.assertEqual(result.binary_sha256, hashlib.sha256(binary).digest()) + self.assertEqual( + result.rebuild_sha256s, + (result.binary_sha256, result.binary_sha256), + ) + self.assertNotEqual( + result.binary_sha256, + _digest("self-reported-output"), + ) + self.assertIs(result.transcript_bytes, run.results[0].stdout) + self.assertEqual( + result.transcript_bytes, + _transcript(result.comparator.identity), + ) + self.assertEqual(result.transcript.encode(), result.transcript_bytes) + self.assertEqual(result.run_claim.binary_identity, result.binary_sha256) + self.assertEqual(result.run_claim.transcript_identity, result.transcript.identity) + self.assertEqual( + result.structural_source_identity, + _request().admitted_sources.identity, + ) + partition = pipeline.flint_source_content_partition_v1( + _request().source_lock, + _request().admitted_sources, + ) + self.assertEqual( + result.flint_commit_content_identity, + partition.commit_content_identity, + ) + self.assertEqual( + result.flint_project_pinned_release_only_identity, + partition.project_pinned_release_only_identity, + ) + self.assertEqual( + result.flint_commit_content_file_count, + partition.commit_content_file_count, + ) + self.assertEqual( + result.flint_project_pinned_release_only_file_count, + partition.project_pinned_release_only_file_count, + ) + self.assertEqual(result.build_input_identity, _build_sources().build_input_identity) + self.assertEqual( + result.formula_support_identity, + _build_sources().formula_support_identity, + ) + self.assertEqual( + result.pipeline_policy_identity, + pipeline.pipeline_policy_identity_v1(result.host_trust), + ) + self.assertFalse(hasattr(result, "build_observer_kind")) + self.assertFalse(hasattr(result, "run_observer_kind")) + self.assertFalse(hasattr(result, "build_source_identity")) + self.assertFalse(hasattr(result, "build_policy_identity")) + self.assertFalse(hasattr(result, "commit_derived_source_identity")) + self.assertEqual( + result.host_trust, + pipeline.HostTrustBoundaryV1.UNSEALED_LINUX_X64_DOCKER_HOST, + ) + self.assertEqual(result.oci_image_reference, pipeline.OCI_IMAGE_REFERENCE_V1) + self.assertEqual(result.oci_platform, pipeline.OCI_PLATFORM_V1) + self.assertFalse(hasattr(result, "slsa_level")) + self.assertFalse(hasattr(result, "fresh_vm")) + + def test_builds_must_be_byte_identical_before_any_run(self) -> None: + first = _static_elf(b"first") + second = _static_elf(b"second") + run = _Executor() + + result = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((first, second)), + execution_controller=run, + ).execute(_request()) + + self.assertEqual( + result, + pipeline.NonReproducibleBuildV1( + hashlib.sha256(first).digest(), + hashlib.sha256(second).digest(), + ), + ) + self.assertEqual(run.requests, []) + + def test_build_input_mutation_or_symlink_output_is_typed_failure(self) -> None: + binary = _static_elf() + cases = ( + ( + _BuildBackend((binary,), mutate_inputs=True), + pipeline.BuildFailureReasonV1.INPUT_CHANGED, + ), + ( + _BuildBackend((binary,), hardlink_input=True), + pipeline.BuildFailureReasonV1.INPUT_CHANGED, + ), + ( + _BuildBackend((binary,), symlink_output=True), + pipeline.BuildFailureReasonV1.INVALID_OUTPUT, + ), + ) + for backend, reason in cases: + with self.subTest(reason=reason): + run = _Executor() + result = pipeline.ControlledPipelineV1( + build_backend=backend, + execution_controller=run, + ).execute(_request()) + self.assertIs(type(result), pipeline.BuildRejectedV1) + self.assertEqual(result.attempt, 1) + self.assertEqual(result.reason, reason) + self.assertEqual(run.requests, []) + + def test_workspace_materialization_collision_is_a_typed_failure(self) -> None: + run = _Executor() + controlled = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((_static_elf(),)), + execution_controller=run, + ) + + with mock.patch.object( + pipeline, + "_write_exact_file", + side_effect=pipeline._TreeMismatchV1("parent collision"), + ): + result = controlled.execute(_request()) + + self.assertIs(type(result), pipeline.BuildRejectedV1) + self.assertEqual(result.attempt, 1) + self.assertEqual( + result.reason, + pipeline.BuildFailureReasonV1.BACKEND_CONTRACT, + ) + self.assertEqual(run.requests, []) + + def test_docker_inability_to_observe_build_edge_is_a_design_blocker(self) -> None: + build = _BuildBackend( + (), + probe=pipeline.DockerUnsupportedV1( + pipeline.DockerBlockerReasonV1.SAME_OBJECT_OUTPUT_UNAVAILABLE, + "post-exit owned-byte observation unavailable", + ), + ) + run = _Executor() + + result = pipeline.ControlledPipelineV1( + build_backend=build, + execution_controller=run, + ).execute(_request()) + + self.assertEqual( + result, + pipeline.PipelineBlockedV1( + pipeline.DockerBlockerReasonV1.SAME_OBJECT_OUTPUT_UNAVAILABLE, + "post-exit owned-byte observation unavailable", + ), + ) + self.assertEqual(build.requests, []) + self.assertEqual(run.requests, []) + + def test_job_that_exceeds_exact_run_limits_is_rejected_before_build(self) -> None: + with self.assertRaises(pipeline.PipelineInputErrorV1) as caught: + _request( + execution_limits=replace( + _limits(), + max_stdin_bytes=1, + ) + ) + + self.assertEqual( + caught.exception.reason, + pipeline.PipelineInputReasonV1.EXECUTION_LIMIT_MISMATCH, + ) + + def test_snapshot_modes_are_normalized_independently_of_host_umask(self) -> None: + binary = _static_elf(b"umask-independent") + previous = os.umask(0o077) + try: + result = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((binary, binary)), + execution_controller=_Executor(), + ).execute(_request()) + finally: + os.umask(previous) + + self.assertIs(type(result), pipeline.DiagnosticPipelineObservationV1) + + def test_binary_digest_from_executor_must_match_the_owned_build_object(self) -> None: + binary = _static_elf() + run = _Executor( + lambda _request: executor.CompletedV1( + _digest("foreign-binary"), + _transcript(), + b"", + ) + ) + + result = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((binary, binary)), + execution_controller=run, + ).execute(_request()) + + self.assertIs(type(result), pipeline.ExecutionRejectedV1) + self.assertEqual(result.reason, pipeline.ExecutionFailureReasonV1.BINARY_MISMATCH) + + def test_only_completed_empty_stderr_canonical_bound_transcript_is_admitted(self) -> None: + binary = _static_elf() + foreign = bytearray(_transcript()) + foreign[16] ^= 1 + cases = ( + ( + lambda request: executor.ExitNonZeroV1( + hashlib.sha256(request.executable).digest(), b"", b"failed", 7 + ), + pipeline.ExecutionRejectedV1, + ), + ( + lambda request: executor.CompletedV1( + hashlib.sha256(request.executable).digest(), _transcript(), b"warning" + ), + pipeline.ExecutionRejectedV1, + ), + ( + lambda request: executor.CompletedV1( + hashlib.sha256(request.executable).digest(), bytes(foreign), b"" + ), + pipeline.TranscriptRejectedV1, + ), + ) + for factory, expected_type in cases: + with self.subTest(expected_type=expected_type): + result = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((binary, binary)), + execution_controller=_Executor(factory), + ).execute(_request()) + self.assertIs(type(result), expected_type) + + def test_controller_derives_exact_invocation_without_backend_metadata(self) -> None: + binary = _static_elf() + run = _Executor() + request = _request() + + result = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((binary, binary)), + execution_controller=run, + ).execute(request) + + self.assertIs(type(result), pipeline.DiagnosticPipelineObservationV1) + invocation = run.requests[0] + self.assertEqual( + invocation.argv, + ( + b"arb-evaluator", + b"--manifest-identity", + result.comparator.identity.hex().encode("ascii"), + b"--job", + b"/dev/stdin", + ), + ) + self.assertEqual(invocation.environment, ((b"LC_ALL", b"C"), (b"TZ", b"UTC"))) + self.assertEqual(invocation.cwd, b"/") + self.assertEqual(invocation.stdin, request.job.encode()) + self.assertEqual(invocation.umask, 0o077) + self.assertEqual( + result.invocation_identity, + pipeline.invocation_identity_v1(invocation), + ) + + def test_pipeline_exports_no_receipt_or_self_report_admission_api(self) -> None: + names = dir(pipeline) + self.assertFalse(any("Receipt" in name for name in names)) + self.assertFalse(hasattr(pipeline.ControlledPipelineV1, "admit_report")) + self.assertFalse(hasattr(pipeline.ControlledPipelineV1, "mint")) + + def test_fake_executor_wrapper_or_native_subclass_stays_diagnostic(self) -> None: + binary = _static_elf(b"run-kind") + wrapped = _MasqueradingControlledExecutor( + _MasqueradingNativeBackend() + ) + exact_executor_with_fake_native = executor.ControlledExecutorV1( + _MasqueradingNativeBackend() + ) + for run in (_Executor(), wrapped, exact_executor_with_fake_native): + with self.subTest(executor_type=type(run).__name__): + result = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((binary, binary)), + execution_controller=run, + ).execute(_request()) + + self.assertIs(type(result), pipeline.DiagnosticPipelineObservationV1) + self.assertFalse(hasattr(result, "run_observer_kind")) + + def test_native_observer_promotion_is_not_representable_in_v1(self) -> None: + for name in ( + "BuildObserverKindV1", + "RunObserverKindV1", + "build_observer_kind_v1", + "run_observer_kind_v1", + "NativePipelineObservationV1", + ): + with self.subTest(name=name): + self.assertFalse(hasattr(pipeline, name)) + + def test_self_mutating_executor_cannot_upgrade_fabricated_run(self) -> None: + binary = _static_elf(b"self-mutating-run") + backend = _SelfMutatingExecutionBackend() + run = executor.ControlledExecutorV1(backend) + backend.owner = run + + result = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((binary, binary)), + execution_controller=run, + ).execute(_request()) + + self.assertIs(type(result), pipeline.DiagnosticPipelineObservationV1) + self.assertFalse(hasattr(result, "run_observer_kind")) + + def test_mutable_exact_native_build_backend_cannot_upgrade_fabricated_build(self) -> None: + binary = _static_elf(b"self-mutating-build") + backend = pipeline.NativeDockerBuildBackendV1( + Path("/bin/true"), + platform_name="linux", + machine_name="x86_64", + ) + + def probe(_self: object) -> pipeline.DockerCapabilityReportV1: + return pipeline.DockerSupportedV1( + pipeline.OCI_IMAGE_REFERENCE_V1, + pipeline.OCI_PLATFORM_V1, + _digest("fabricated-daemon"), + ) + + def run_build( + _self: object, + request: pipeline.DockerBuildRequestV1, + ) -> pipeline.DockerBuildProcessObservationV1: + target = request.output_directory / pipeline.EVALUATOR_OUTPUT_NAME_V1 + target.write_bytes(binary) + target.chmod(0o555) + return pipeline.DockerBuildExitedV1(0, b"self-reported-native", b"") + + backend.probe = MethodType(probe, backend) + backend.run_build = MethodType(run_build, backend) + + result = pipeline.ControlledPipelineV1( + build_backend=backend, + execution_controller=_Executor(), + ).execute(_request()) + + self.assertIs(type(result), pipeline.DiagnosticPipelineObservationV1) + self.assertFalse(hasattr(result, "build_observer_kind")) + + +class DockerCommandContractTests(unittest.TestCase): + def test_command_is_exact_digest_offline_read_only_and_capability_free(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + directories = tuple( + root / name for name in ("inputs", "workspace", "build", "out") + ) + for directory in directories: + directory.mkdir() + request = pipeline.DockerBuildRequestV1( + 1, + root, + *directories, + root / "container.cid", + "labcolors-arb-build-v1-test", + ) + backend = pipeline.NativeDockerBuildBackendV1( + Path("/usr/bin/docker"), + platform_name="linux", + machine_name="x86_64", + ) + + command = backend.command_for(request) + + joined = " ".join(command) + self.assertEqual(command[0], "/usr/bin/docker") + self.assertIn(pipeline.OCI_IMAGE_REFERENCE_V1, command) + self.assertNotIn("gcc:latest", joined) + for fragment in ( + "--pull never", + "--platform linux/amd64", + "--network none", + "--read-only", + "--cap-drop ALL", + "--security-opt no-new-privileges:true", + "--name labcolors-arb-build-v1-test", + "readonly,bind-propagation=private", + "dst=/inputs", + "dst=/workspace", + "dst=/build", + "dst=/out", + "--rm", + ): + with self.subTest(fragment=fragment): + self.assertIn(fragment, joined) + for forbidden in ("--privileged", "--network host", ":latest"): + self.assertNotIn(forbidden, joined) + + def test_command_exposes_only_a_private_non_executable_standard_tmp(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + directories = tuple(root / name for name in ("inputs", "workspace", "build", "out")) + for directory in directories: + directory.mkdir() + request = pipeline.DockerBuildRequestV1( + 1, + root, + *directories, + root / "container.cid", + "labcolors-arb-build-v1-test", + ) + command = pipeline.NativeDockerBuildBackendV1( + Path("/usr/bin/docker"), + platform_name="linux", + machine_name="x86_64", + ).command_for(request) + + tmpfs_indexes = tuple( + index for index, item in enumerate(command) if item == "--tmpfs" + ) + self.assertEqual(len(tmpfs_indexes), 1) + self.assertEqual( + command[tmpfs_indexes[0] + 1], + "/tmp:rw,noexec,nosuid,nodev,mode=1777", + ) + self.assertNotIn("src=", command[tmpfs_indexes[0] + 1]) + mount_indexes = tuple( + index for index, item in enumerate(command) if item == "--mount" + ) + self.assertEqual(len(mount_indexes), 4) + mount_specs = tuple(command[index + 1] for index in mount_indexes) + mount_destinations = tuple( + item.removeprefix("dst=") + for spec in mount_specs + for item in spec.split(",") + if item.startswith("dst=") + ) + self.assertEqual( + mount_destinations, + ("/inputs", "/workspace", "/build", "/out"), + ) + self.assertTrue(all("dst=/tmp" not in spec for spec in mount_specs)) + self.assertNotIn("-v", command) + self.assertNotIn("--volume", command) + + def test_native_probe_fails_closed_without_linux_or_exact_docker(self) -> None: + non_linux = pipeline.NativeDockerBuildBackendV1( + Path("/usr/bin/docker"), + platform_name="darwin", + machine_name="arm64", + ).probe() + missing = pipeline.NativeDockerBuildBackendV1( + Path("/definitely/missing/docker"), + platform_name="linux", + machine_name="x86_64", + ).probe() + + self.assertEqual(non_linux.reason, pipeline.DockerBlockerReasonV1.HOST_NOT_LINUX_AMD64) + self.assertEqual(missing.reason, pipeline.DockerBlockerReasonV1.DOCKER_UNAVAILABLE) + + def test_native_command_observer_caps_probe_output_before_allocation(self) -> None: + backend = pipeline.NativeDockerBuildBackendV1( + Path("/bin/sh"), + platform_name="linux", + machine_name="x86_64", + ) + + result = backend._observe_command( + ( + sys.executable, + "-c", + "import os; os.write(1, b'x' * 65536)", + ), + stdout_limit=8, + stderr_limit=8, + timeout_ns=5_000_000_000, + cid_file=None, + ) + + self.assertEqual( + result, + pipeline.DockerBuildOutputLimitV1( + pipeline.DockerOutputStreamV1.STDOUT, + b"x" * 8, + b"", + ), + ) + + def test_cleanup_falls_back_to_exact_name_for_absent_or_invalid_cidfile(self) -> None: + backend = pipeline.NativeDockerBuildBackendV1( + Path("/bin/sh"), + platform_name="linux", + machine_name="x86_64", + ) + name = "labcolors-arb-build-v1-cleanup-test" + for cid_contents in (None, b"partial-or-foreign"): + with self.subTest(cid_contents=cid_contents): + with tempfile.TemporaryDirectory() as temporary: + cid_file = Path(temporary) / "container.cid" + if cid_contents is not None: + cid_file.write_bytes(cid_contents) + observations = ( + pipeline.DockerBuildExitedV1(1, b"", b"not found"), + pipeline.DockerBuildExitedV1(0, b"", b""), + ) + with mock.patch.object( + backend, + "_observe_cleanup_command", + side_effect=observations, + ) as observe: + detail = backend._cleanup_container(cid_file, name) + + self.assertIsNone(detail) + commands = tuple(call.args[0] for call in observe.call_args_list) + self.assertEqual(commands[0][-1], name) + self.assertIn(f"name=^/{name}$", commands[1]) + self.assertNotIn("partial-or-foreign", " ".join(commands[0])) + + def test_unverified_container_removal_is_typed_cleanup_failure(self) -> None: + backend = pipeline.NativeDockerBuildBackendV1( + Path("/bin/sh"), + platform_name="linux", + machine_name="x86_64", + ) + with tempfile.TemporaryDirectory() as temporary: + cid_file = Path(temporary).resolve() / "container.cid" + with mock.patch.object( + backend, + "_cleanup_container", + return_value="container absence could not be verified", + ): + result = backend._observe_command( + (sys.executable, "-c", "pass"), + stdout_limit=8, + stderr_limit=8, + timeout_ns=5_000_000_000, + cid_file=cid_file, + container_name="labcolors-arb-build-v1-cleanup-failure", + ) + + self.assertIs(type(result), pipeline.DockerBuildCleanupFailureV1) + self.assertEqual( + result.trigger, + pipeline.DockerCleanupTriggerV1.PROCESS_EXIT, + ) + + +@unittest.skipUnless( + sys.platform == "linux" + and os.environ.get("LABCOLORS_ARB_PIPELINE_DOCKER") + and os.environ.get("LABCOLORS_ARB_NATIVE_BINARY") + and os.environ.get("LABCOLORS_GMP_ARCHIVE") + and os.environ.get("LABCOLORS_MPFR_ARCHIVE") + and os.environ.get("LABCOLORS_FLINT_ARCHIVE"), + "requires Linux, Docker, the native binary path, and all three exact source archives", +) +class NativeBuildIntegrationTests(unittest.TestCase): + def test_real_two_builds_and_ephemeral_evaluator_runtime_tests(self) -> None: + source_lock = provenance.arb_source_lock_v1() + archive_names = ( + "LABCOLORS_GMP_ARCHIVE", + "LABCOLORS_MPFR_ARCHIVE", + "LABCOLORS_FLINT_ARCHIVE", + ) + safe = tuple( + provenance.admit_source_archive(lock, Path(os.environ[name]).read_bytes()) + for lock, name in zip(source_lock.sources, archive_names, strict=True) + ) + admitted = provenance.admit_arb_sources(source_lock, safe) + controller = pipeline.ControlledPipelineV1( + build_backend=pipeline.NativeDockerBuildBackendV1( + Path(os.environ["LABCOLORS_ARB_PIPELINE_DOCKER"]) + ), + execution_controller=None, + ) + + result = controller.build( + _request(source_lock=source_lock, admitted_sources=admitted) + ) + + self.assertIs(type(result), pipeline.DiagnosticBuildObservationV1, result) + self.assertEqual(result.rebuild_sha256s, (result.binary_sha256,) * 2) + + # This executable is deliberately ephemeral and is never uploaded: a + # distributable static artifact needs a separate linker/legal gate. + with tempfile.TemporaryDirectory(prefix="labcolors-arb-evaluator-tests-") as temporary: + executable = Path(temporary) / pipeline.EVALUATOR_OUTPUT_NAME_V1 + executable.write_bytes(result.binary) + executable.chmod(0o555) + environment = { + "LABCOLORS_ARB_EVALUATOR": str(executable), + "LC_ALL": "C", + "PATH": os.environ.get("PATH", ""), + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHASHSEED": "0", + "TZ": "UTC", + } + runtime = subprocess.run( + ( + sys.executable, + str( + REPO + / "proof/region/v1/arb/tests/runtime_gate.py" + ), + ), + check=False, + capture_output=True, + cwd=REPO, + env=environment, + timeout=300, + ) + + self.assertEqual( + runtime.returncode, + 0, + (runtime.stdout + runtime.stderr).decode("utf-8", "replace"), + ) + binary_path = Path(os.environ["LABCOLORS_ARB_NATIVE_BINARY"]) + self.assertTrue(binary_path.is_absolute()) + descriptor = os.open( + binary_path, + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | os.O_CLOEXEC + | getattr(os, "O_NOFOLLOW", 0), + 0o400, + ) + try: + view = memoryview(result.binary) + offset = 0 + while offset < len(view): + try: + written = os.write(descriptor, view[offset:]) + except InterruptedError: + continue + if written <= 0: + raise OSError("short native binary write") + offset += written + os.fsync(descriptor) + finally: + os.close(descriptor) + + +@unittest.skipUnless( + sys.platform == "linux" + and os.environ.get("LABCOLORS_ARB_NATIVE_BINARY") + and os.environ.get("LABCOLORS_EXECUTOR_CGROUP_V1"), + "requires Linux, the native binary path, and an explicit delegated " + "cgroup v2 parent", +) +class NativePipelineIntegrationTests(unittest.TestCase): + def test_prepared_two_build_binary_runs_through_controlled_pipeline(self) -> None: + binary = Path(os.environ["LABCOLORS_ARB_NATIVE_BINARY"]).read_bytes() + request = _request() + controlled = pipeline.ControlledPipelineV1( + build_backend=_BuildBackend((binary, binary)), + execution_controller=executor.ControlledExecutorV1( + executor.NativeLinuxBackendV1( + Path(os.environ["LABCOLORS_EXECUTOR_CGROUP_V1"]) + ) + ), + ) + + result = controlled.execute(request) + + self.assertIs(type(result), pipeline.DiagnosticPipelineObservationV1, result) + self.assertEqual(result.build_observation.binary, binary) + self.assertEqual( + result.build_observation.binary_sha256, + hashlib.sha256(binary).digest(), + ) + self.assertEqual(result.transcript.job_identity, request.job.identity) + self.assertEqual( + result.transcript.comparator_identity, + result.comparator.identity, + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/proof/region/v1/arb/tests/test_snapshot.py b/proof/region/v1/arb/tests/test_snapshot.py new file mode 100644 index 00000000..7e1e16f8 --- /dev/null +++ b/proof/region/v1/arb/tests/test_snapshot.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Hostile tests for normalized source snapshots.""" + +from __future__ import annotations + +import gzip +import hashlib +import io +import stat +import sys +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +PROOF = Path(__file__).resolve().parents[2] +ARB = PROOF / "arb" +sys.path.insert(0, str(PROOF)) +sys.path.insert(0, str(ARB)) + +import provenance # noqa: E402 +import snapshot # noqa: E402 + + +def fixture() -> tuple[provenance.SourceReleaseLockV1, bytes]: + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode="w", format=tarfile.USTAR_FORMAT) as archive: + for name in ("fixture-1/", "fixture-1/src/"): + member = tarfile.TarInfo(name) + member.type = tarfile.DIRTYPE + member.mode = 0o755 + member.mtime = 0 + archive.addfile(member) + for name, body, mode in ( + ("fixture-1/LICENSE", b"license", 0o644), + ("fixture-1/src/tool", b"tool", 0o755), + ): + member = tarfile.TarInfo(name) + member.mode = mode + member.size = len(body) + member.mtime = 0 + archive.addfile(member, io.BytesIO(body)) + archive_bytes = gzip.compress(raw.getvalue(), compresslevel=9, mtime=0) + lock = provenance.SourceReleaseLockV1( + provenance.SourceRoleV1.GMP, + "1", + "https://example.invalid/fixture-1.tar.gz", + provenance.ArchiveFormatV1.TAR_GZIP, + len(archive_bytes), + hashlib.sha256(archive_bytes).digest(), + len(raw.getvalue()), + "fixture-1/", + 2, + 11, + ( + provenance.LegalFileV1( + "LICENSE", 7, hashlib.sha256(b"license").digest() + ), + ), + provenance.DetachedSignaturePolicyV1( + "https://example.invalid/fixture-1.tar.gz.sig", + 3, + hashlib.sha256(b"sig").digest(), + hashlib.sha256(b"packets").digest(), + bytes.fromhex("11" * 20), + ), + ) + return lock, archive_bytes + + +class SourceSnapshotTests(unittest.TestCase): + def test_snapshot_depends_only_on_public_provenance_surface(self) -> None: + source = (ARB / "snapshot.py").read_text(encoding="utf-8") + + self.assertNotIn("provenance._", source) + + def test_only_admitted_regular_files_materialize_with_exact_modes(self) -> None: + lock, archive_bytes = fixture() + admitted = provenance.admit_source_archive(lock, archive_bytes) + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "fixture-1" + result = snapshot.materialize_source_archive(lock, admitted, destination) + + self.assertEqual(result.tree_identity, admitted.tree_identity) + self.assertEqual(result.regular_file_count, 2) + self.assertEqual((destination / "LICENSE").read_bytes(), b"license") + self.assertEqual((destination / "src/tool").read_bytes(), b"tool") + self.assertEqual(stat.S_IMODE((destination / "LICENSE").stat().st_mode), 0o644) + self.assertEqual(stat.S_IMODE((destination / "src/tool").stat().st_mode), 0o755) + for path in ( + destination, + destination / "LICENSE", + destination / "src", + destination / "src/tool", + ): + with self.subTest(path=path): + self.assertEqual( + path.stat().st_mtime_ns, + snapshot.SOURCE_SNAPSHOT_MTIME_NS_V1, + ) + + def test_materialization_decompresses_the_owned_archive_once(self) -> None: + lock, archive_bytes = fixture() + admitted = provenance.admit_source_archive(lock, archive_bytes) + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "fixture-1" + with mock.patch.object( + provenance, + "_decompress_exact", + wraps=provenance._decompress_exact, + ) as decompress: + snapshot.materialize_source_archive(lock, admitted, destination) + + self.assertEqual(decompress.call_count, 1) + + def test_single_pass_replay_rejects_capability_coordinate_drift(self) -> None: + lock, archive_bytes = fixture() + original = provenance.admit_source_archive(lock, archive_bytes) + mutations = ( + ("archive_sha256", bytes.fromhex("ff" * 32)), + ("tree_identity", bytes.fromhex("ff" * 32)), + ("regular_file_count", original.regular_file_count + 1), + ("regular_file_bytes", original.regular_file_bytes + 1), + ("files", original.files[:-1]), + ) + for field, value in mutations: + with self.subTest(field=field): + with tempfile.TemporaryDirectory() as temporary: + admitted = provenance.admit_source_archive(lock, archive_bytes) + object.__setattr__(admitted, field, value) + destination = Path(temporary) / "fixture-1" + with mock.patch.object( + provenance, + "_decompress_exact", + wraps=provenance._decompress_exact, + ) as decompress: + with self.assertRaises(snapshot.SnapshotErrorV1) as caught: + snapshot.materialize_source_archive( + lock, + admitted, + destination, + ) + + self.assertEqual( + caught.exception.reason, + snapshot.SnapshotReasonV1.FOREIGN_CAPABILITY, + ) + self.assertEqual(decompress.call_count, 1) + + def test_destination_must_be_new_exact_release_root(self) -> None: + lock, archive_bytes = fixture() + admitted = provenance.admit_source_archive(lock, archive_bytes) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for destination in (root / "wrong", root): + with self.subTest(destination=destination): + with self.assertRaises(snapshot.SnapshotErrorV1): + snapshot.materialize_source_archive(lock, admitted, destination) + + def test_timestamp_normalization_must_verify_the_filesystem_postcondition(self) -> None: + lock, archive_bytes = fixture() + admitted = provenance.admit_source_archive(lock, archive_bytes) + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "fixture-1" + with mock.patch.object(snapshot.os, "utime", return_value=None): + with self.assertRaises(snapshot.SnapshotErrorV1) as caught: + snapshot.materialize_source_archive(lock, admitted, destination) + + self.assertEqual(caught.exception.reason, snapshot.SnapshotReasonV1.IO_FAILURE) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/proof/region/v1/provenance.py b/proof/region/v1/provenance.py new file mode 100644 index 00000000..d9d08800 --- /dev/null +++ b/proof/region/v1/provenance.py @@ -0,0 +1,1186 @@ +#!/usr/bin/env python3 +"""Canonical source declarations and fail-closed archive admission for proof V1. + +This module deliberately stops before cryptographic origin verification, build +execution and evaluator replay. Observations remain claims; only archive bytes +that were hashed and structurally scanned become ``SafeSourceArchiveV1``. +""" + +from __future__ import annotations + +import hashlib +import io +import lzma +import tarfile +import zlib +from dataclasses import dataclass, field +from enum import IntEnum, StrEnum +from functools import cached_property +from pathlib import PurePosixPath +from typing import NoReturn, TypeAlias +from urllib.parse import urlsplit + + +SOURCE_LOCK_MAGIC_V1 = b"LCSRC1\0\0" +SOURCE_LOCK_ID_LABEL_V1 = b"labcolors.proof-region.source-lock.v1\0" +SOURCE_TREE_ID_LABEL_V1 = b"labcolors.proof-region.safe-source-tree.v1\0" +ADMITTED_ARB_SOURCES_ID_LABEL_V1 = b"labcolors.proof-region.admitted-arb-sources.v1\0" +SOURCE_LOCK_RELEASE_V1 = 1 +ARBITRARY_PRECISION_SOURCE_COUNT_V1 = 3 +SHA256_BYTES = 32 +SHA1_BYTES = 20 +OPENPGP_V4_FINGERPRINT_BYTES = 20 +TAR_BLOCK_BYTES = 512 +TAR_END_MARKER_BYTES = TAR_BLOCK_BYTES * 2 +READ_CHUNK_BYTES = 64 * 1024 +ALLOWED_REGULAR_MODES_V1 = frozenset((0o644, 0o700, 0o755)) +ALLOWED_DIRECTORY_MODE_V1 = 0o755 + + +class ProvenanceReasonV1(StrEnum): + BAD_MAGIC = "bad_magic" + TRUNCATED = "truncated" + TRAILING_BYTES = "trailing_bytes" + UNKNOWN_RELEASE = "unknown_release" + UNKNOWN_ENUM = "unknown_enum" + INVALID_FIELD = "invalid_field" + INVALID_DIGEST = "invalid_digest" + NONCANONICAL_ORDER = "noncanonical_order" + DUPLICATE_PATH = "duplicate_path" + CASE_COLLISION = "case_collision" + ABSOLUTE_PATH = "absolute_path" + UNSAFE_PATH = "unsafe_path" + UNSAFE_LINK = "unsafe_link" + UNSAFE_MEMBER_TYPE = "unsafe_member_type" + UNSAFE_MODE = "unsafe_mode" + ARCHIVE_LENGTH_MISMATCH = "archive_length_mismatch" + ARCHIVE_DIGEST_MISMATCH = "archive_digest_mismatch" + DECOMPRESSION_FAILED = "decompression_failed" + TAR_STREAM_LENGTH_MISMATCH = "tar_stream_length_mismatch" + TRAILING_COMPRESSED_DATA = "trailing_compressed_data" + NONCANONICAL_TAR = "noncanonical_tar" + ROOT_MISMATCH = "root_mismatch" + FILE_COUNT_MISMATCH = "file_count_mismatch" + FILE_BYTES_MISMATCH = "file_bytes_mismatch" + FILE_CONTENT_MISMATCH = "file_content_mismatch" + LEGAL_FILES_MISMATCH = "legal_files_mismatch" + CONTENT_RELATION_MISMATCH = "content_relation_mismatch" + FOREIGN_BINDING = "foreign_binding" + INTEGRITY_KIND_MISMATCH = "integrity_kind_mismatch" + + +@dataclass(frozen=True) +class ProvenanceErrorV1(ValueError): + artifact: str + reason: ProvenanceReasonV1 + detail: str + + def __str__(self) -> str: + return f"{self.artifact}: {self.reason}: {self.detail}" + + +def _fail(artifact: str, reason: ProvenanceReasonV1, detail: str) -> NoReturn: + raise ProvenanceErrorV1(artifact, reason, detail) + + +def _identity(label: bytes, encoded: bytes) -> bytes: + return hashlib.sha256(label + len(encoded).to_bytes(8, "big") + encoded).digest() + + +def _digest(value: bytes, artifact: str, field_name: str, length: int = SHA256_BYTES) -> bytes: + if type(value) is not bytes or len(value) != length or value == bytes(length): + _fail(artifact, ProvenanceReasonV1.INVALID_DIGEST, f"invalid {field_name}") + return value + + +def _positive(value: int, artifact: str, field_name: str) -> int: + if type(value) is not int or value <= 0 or value >= 1 << 64: + _fail(artifact, ProvenanceReasonV1.INVALID_FIELD, f"invalid {field_name}") + return value + + +def _ascii(value: str, artifact: str, field_name: str, maximum: int) -> bytes: + if type(value) is not str or not value or "\0" in value: + _fail(artifact, ProvenanceReasonV1.INVALID_FIELD, f"invalid {field_name}") + try: + encoded = value.encode("ascii") + except UnicodeEncodeError: + _fail(artifact, ProvenanceReasonV1.INVALID_FIELD, f"non-ASCII {field_name}") + if any(byte < 0x20 or byte == 0x7F for byte in encoded): + _fail(artifact, ProvenanceReasonV1.INVALID_FIELD, f"control byte in {field_name}") + if len(encoded) > maximum: + _fail(artifact, ProvenanceReasonV1.INVALID_FIELD, f"oversized {field_name}") + return encoded + + +def _relative_path(value: str, artifact: str, field_name: str) -> bytes: + encoded = _ascii(value, artifact, field_name, 4096) + if value.startswith("/"): + _fail(artifact, ProvenanceReasonV1.ABSOLUTE_PATH, f"absolute {field_name}") + if "\\" in value: + _fail(artifact, ProvenanceReasonV1.UNSAFE_PATH, f"backslash in {field_name}") + parts = value.split("/") + if not parts or any(part in ("", ".", "..") for part in parts): + _fail(artifact, ProvenanceReasonV1.UNSAFE_PATH, f"unsafe {field_name}") + return encoded + + +def _root_prefix(value: str, artifact: str) -> bytes: + if type(value) is not str or not value.endswith("/") or value.count("/") != 1: + _fail(artifact, ProvenanceReasonV1.INVALID_FIELD, "root prefix is one directory") + return _relative_path(value[:-1], artifact, "root_prefix") + b"/" + + +def _https_url(value: str, artifact: str, field_name: str) -> bytes: + encoded = _ascii(value, artifact, field_name, 2048) + try: + parsed = urlsplit(value) + hostname = parsed.hostname + _ = parsed.port + except ValueError: + _fail(artifact, ProvenanceReasonV1.INVALID_FIELD, f"malformed {field_name}") + if ( + parsed.scheme != "https" + or not hostname + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + or parsed.path in ("", "/") + ): + _fail(artifact, ProvenanceReasonV1.INVALID_FIELD, f"noncanonical {field_name}") + return encoded + + +def _blob(value: bytes) -> bytes: + return len(value).to_bytes(4, "big") + value + + +class _Reader: + def __init__(self, data: bytes, artifact: str): + if type(data) is not bytes: + raise TypeError("canonical wire input must be bytes") + self.data = data + self.artifact = artifact + self.offset = 0 + + def exact(self, length: int) -> bytes: + if length < 0 or self.offset + length > len(self.data): + _fail(self.artifact, ProvenanceReasonV1.TRUNCATED, "wire is truncated") + start = self.offset + self.offset += length + return self.data[start : self.offset] + + def u8(self) -> int: + return self.exact(1)[0] + + def u16(self) -> int: + return int.from_bytes(self.exact(2), "big") + + def u32(self) -> int: + return int.from_bytes(self.exact(4), "big") + + def u64(self) -> int: + return int.from_bytes(self.exact(8), "big") + + def blob(self, maximum: int) -> bytes: + length = self.u32() + if length == 0 or length > maximum: + _fail(self.artifact, ProvenanceReasonV1.INVALID_FIELD, "invalid blob length") + return self.exact(length) + + def text(self, maximum: int, field_name: str) -> str: + raw = self.blob(maximum) + try: + return raw.decode("ascii") + except UnicodeDecodeError: + _fail(self.artifact, ProvenanceReasonV1.INVALID_FIELD, f"non-ASCII {field_name}") + + def finish(self) -> None: + if self.offset != len(self.data): + _fail(self.artifact, ProvenanceReasonV1.TRAILING_BYTES, "wire has trailing bytes") + + +class ArchiveFormatV1(IntEnum): + TAR_XZ = 1 + TAR_GZIP = 2 + + +class SourceRoleV1(IntEnum): + GMP = 1 + MPFR = 2 + FLINT_ARB = 3 + + +class IntegrityKindV1(IntEnum): + DETACHED_SIGNATURE = 1 + GIT_CONTENT_RELATION = 2 + + +@dataclass(frozen=True) +class LegalFileV1: + path: str + length: int + sha256: bytes + + def __post_init__(self) -> None: + _relative_path(self.path, "legal-file-v1", "path") + _positive(self.length, "legal-file-v1", "length") + _digest(self.sha256, "legal-file-v1", "sha256") + + def encode(self) -> bytes: + return _blob(self.path.encode("ascii")) + self.length.to_bytes(8, "big") + self.sha256 + + @classmethod + def parse_from(cls, reader: _Reader) -> "LegalFileV1": + return cls(reader.text(4096, "legal file path"), reader.u64(), reader.exact(SHA256_BYTES)) + + +@dataclass(frozen=True) +class ProjectPinnedReleaseOnlyFileV1: + path: str + mode: int + length: int + sha256: bytes + + def __post_init__(self) -> None: + _relative_path(self.path, "project-pinned-release-only-file-v1", "path") + if type(self.mode) is not int or self.mode not in ALLOWED_REGULAR_MODES_V1: + _fail( + "project-pinned-release-only-file-v1", + ProvenanceReasonV1.UNSAFE_MODE, + "invalid mode", + ) + _positive(self.length, "project-pinned-release-only-file-v1", "length") + _digest(self.sha256, "project-pinned-release-only-file-v1", "sha256") + + def encode(self) -> bytes: + return ( + _blob(self.path.encode("ascii")) + + self.mode.to_bytes(4, "big") + + self.length.to_bytes(8, "big") + + self.sha256 + ) + + @classmethod + def parse_from(cls, reader: _Reader) -> "ProjectPinnedReleaseOnlyFileV1": + return cls( + reader.text(4096, "project-pinned release-only path"), + reader.u32(), + reader.u64(), + reader.exact(SHA256_BYTES), + ) + + +@dataclass(frozen=True) +class DetachedSignaturePolicyV1: + signature_url: str + signature_length: int + signature_sha256: bytes + public_key_packets_sha256: bytes + signer_fingerprint: bytes + + kind: IntegrityKindV1 = field( + init=False, + default=IntegrityKindV1.DETACHED_SIGNATURE, + ) + + def __post_init__(self) -> None: + _https_url(self.signature_url, "signature-policy-v1", "signature_url") + _positive(self.signature_length, "signature-policy-v1", "signature_length") + _digest(self.signature_sha256, "signature-policy-v1", "signature_sha256") + _digest( + self.public_key_packets_sha256, + "signature-policy-v1", + "public_key_packets_sha256", + ) + _digest( + self.signer_fingerprint, + "signature-policy-v1", + "signer_fingerprint", + OPENPGP_V4_FINGERPRINT_BYTES, + ) + + def encode_payload(self) -> bytes: + return ( + _blob(self.signature_url.encode("ascii")) + + self.signature_length.to_bytes(8, "big") + + self.signature_sha256 + + self.public_key_packets_sha256 + + self.signer_fingerprint + ) + + @classmethod + def parse_from(cls, reader: _Reader) -> "DetachedSignaturePolicyV1": + return cls( + reader.text(2048, "signature URL"), + reader.u64(), + reader.exact(SHA256_BYTES), + reader.exact(SHA256_BYTES), + reader.exact(OPENPGP_V4_FINGERPRINT_BYTES), + ) + + +@dataclass(frozen=True) +class GitContentRelationPolicyV1: + repository_url: str + tag: str + commit: bytes + tree: bytes + common_file_count: int + omitted_paths: tuple[str, ...] + project_pinned_release_only_files: tuple[ProjectPinnedReleaseOnlyFileV1, ...] + + kind: IntegrityKindV1 = field( + init=False, + default=IntegrityKindV1.GIT_CONTENT_RELATION, + ) + + def __post_init__(self) -> None: + artifact = "git-content-relation-policy-v1" + _https_url(self.repository_url, artifact, "repository_url") + _ascii(self.tag, artifact, "tag", 128) + _digest(self.commit, artifact, "commit", SHA1_BYTES) + _digest(self.tree, artifact, "tree", SHA1_BYTES) + _positive(self.common_file_count, artifact, "common_file_count") + if ( + type(self.omitted_paths) is not tuple + or not self.omitted_paths + or len(self.omitted_paths) > 4096 + ): + _fail(artifact, ProvenanceReasonV1.INVALID_FIELD, "omission count") + if ( + type(self.project_pinned_release_only_files) is not tuple + or not self.project_pinned_release_only_files + or len(self.project_pinned_release_only_files) > 4096 + ): + _fail(artifact, ProvenanceReasonV1.INVALID_FIELD, "release-only file count") + for path in self.omitted_paths: + _relative_path(path, artifact, "omitted path") + if self.omitted_paths != tuple(sorted(set(self.omitted_paths))): + _fail(artifact, ProvenanceReasonV1.NONCANONICAL_ORDER, "omissions") + if any( + type(value) is not ProjectPinnedReleaseOnlyFileV1 + for value in self.project_pinned_release_only_files + ): + _fail(artifact, ProvenanceReasonV1.INVALID_FIELD, "release-only file type") + release_only_paths = tuple( + value.path for value in self.project_pinned_release_only_files + ) + if release_only_paths != tuple(sorted(set(release_only_paths))): + _fail( + artifact, + ProvenanceReasonV1.NONCANONICAL_ORDER, + "project-pinned release-only files", + ) + if set(release_only_paths) & set(self.omitted_paths): + _fail(artifact, ProvenanceReasonV1.INVALID_FIELD, "relation overlap") + + def encode_payload(self) -> bytes: + chunks = [ + _blob(self.repository_url.encode("ascii")), + _blob(self.tag.encode("ascii")), + self.commit, + self.tree, + self.common_file_count.to_bytes(8, "big"), + len(self.omitted_paths).to_bytes(4, "big"), + ] + chunks.extend(_blob(path.encode("ascii")) for path in self.omitted_paths) + chunks.append(len(self.project_pinned_release_only_files).to_bytes(4, "big")) + chunks.extend(value.encode() for value in self.project_pinned_release_only_files) + return b"".join(chunks) + + @classmethod + def parse_from(cls, reader: _Reader) -> "GitContentRelationPolicyV1": + repository = reader.text(2048, "repository URL") + tag = reader.text(128, "tag") + commit = reader.exact(SHA1_BYTES) + tree = reader.exact(SHA1_BYTES) + common_file_count = reader.u64() + omitted_count = reader.u32() + if omitted_count == 0 or omitted_count > 4096: + _fail(reader.artifact, ProvenanceReasonV1.INVALID_FIELD, "omission count") + omitted = tuple(reader.text(4096, "omitted path") for _ in range(omitted_count)) + release_only_count = reader.u32() + if release_only_count == 0 or release_only_count > 4096: + _fail(reader.artifact, ProvenanceReasonV1.INVALID_FIELD, "release-only file count") + release_only = tuple( + ProjectPinnedReleaseOnlyFileV1.parse_from(reader) + for _ in range(release_only_count) + ) + return cls(repository, tag, commit, tree, common_file_count, omitted, release_only) + + +SourceIntegrityPolicyV1: TypeAlias = ( + DetachedSignaturePolicyV1 | GitContentRelationPolicyV1 +) + + +def _parse_integrity_policy(reader: _Reader) -> SourceIntegrityPolicyV1: + kind_value = reader.u8() + try: + kind = IntegrityKindV1(kind_value) + except ValueError: + _fail(reader.artifact, ProvenanceReasonV1.UNKNOWN_ENUM, "integrity kind") + if kind is IntegrityKindV1.DETACHED_SIGNATURE: + return DetachedSignaturePolicyV1.parse_from(reader) + return GitContentRelationPolicyV1.parse_from(reader) + + +@dataclass(frozen=True) +class SourceReleaseLockV1: + role: SourceRoleV1 + version: str + archive_url: str + archive_format: ArchiveFormatV1 + archive_length: int + archive_sha256: bytes + tar_stream_length: int + root_prefix: str + regular_file_count: int + regular_file_bytes: int + legal_files: tuple[LegalFileV1, ...] + integrity: SourceIntegrityPolicyV1 + + def __post_init__(self) -> None: + if type(self.role) is not SourceRoleV1 or type(self.archive_format) is not ArchiveFormatV1: + _fail("source-release-lock-v1", ProvenanceReasonV1.UNKNOWN_ENUM, "role or format") + _ascii(self.version, "source-release-lock-v1", "version", 128) + _https_url(self.archive_url, "source-release-lock-v1", "archive_url") + _positive(self.archive_length, "source-release-lock-v1", "archive_length") + _digest(self.archive_sha256, "source-release-lock-v1", "archive_sha256") + _positive(self.tar_stream_length, "source-release-lock-v1", "tar_stream_length") + if self.tar_stream_length % TAR_BLOCK_BYTES: + _fail("source-release-lock-v1", ProvenanceReasonV1.INVALID_FIELD, "unaligned tar length") + _root_prefix(self.root_prefix, "source-release-lock-v1") + _positive(self.regular_file_count, "source-release-lock-v1", "regular_file_count") + _positive(self.regular_file_bytes, "source-release-lock-v1", "regular_file_bytes") + if ( + type(self.legal_files) is not tuple + or not self.legal_files + or len(self.legal_files) > 4096 + ): + _fail("source-release-lock-v1", ProvenanceReasonV1.INVALID_FIELD, "legal file count") + if any(type(value) is not LegalFileV1 for value in self.legal_files): + _fail("source-release-lock-v1", ProvenanceReasonV1.INVALID_FIELD, "legal file type") + paths = tuple(value.path for value in self.legal_files) + if paths != tuple(sorted(set(paths))): + _fail("source-release-lock-v1", ProvenanceReasonV1.NONCANONICAL_ORDER, "legal files") + if type(self.integrity) not in ( + DetachedSignaturePolicyV1, + GitContentRelationPolicyV1, + ): + _fail( + "source-release-lock-v1", + ProvenanceReasonV1.UNKNOWN_ENUM, + "integrity policy", + ) + if isinstance(self.integrity, GitContentRelationPolicyV1): + if ( + self.integrity.common_file_count + + len(self.integrity.project_pinned_release_only_files) + != self.regular_file_count + ): + _fail( + "source-release-lock-v1", + ProvenanceReasonV1.CONTENT_RELATION_MISMATCH, + "common plus project-pinned release-only count does not cover archive", + ) + + def encode(self) -> bytes: + chunks = [ + bytes((self.role,)), + _blob(self.version.encode("ascii")), + _blob(self.archive_url.encode("ascii")), + bytes((self.archive_format,)), + self.archive_length.to_bytes(8, "big"), + self.archive_sha256, + self.tar_stream_length.to_bytes(8, "big"), + _blob(self.root_prefix.encode("ascii")), + self.regular_file_count.to_bytes(8, "big"), + self.regular_file_bytes.to_bytes(8, "big"), + len(self.legal_files).to_bytes(2, "big"), + ] + chunks.extend(value.encode() for value in self.legal_files) + chunks.append(bytes((self.integrity.kind,))) + chunks.append(self.integrity.encode_payload()) + return b"".join(chunks) + + @classmethod + def parse_from(cls, reader: _Reader) -> "SourceReleaseLockV1": + role_value = reader.u8() + try: + role = SourceRoleV1(role_value) + except ValueError: + _fail(reader.artifact, ProvenanceReasonV1.UNKNOWN_ENUM, "source role") + version = reader.text(128, "version") + archive_url = reader.text(2048, "archive URL") + archive_format_value = reader.u8() + try: + archive_format = ArchiveFormatV1(archive_format_value) + except ValueError: + _fail(reader.artifact, ProvenanceReasonV1.UNKNOWN_ENUM, "archive format") + archive_length = reader.u64() + archive_sha256 = reader.exact(SHA256_BYTES) + tar_stream_length = reader.u64() + root_prefix = reader.text(4096, "root prefix") + regular_file_count = reader.u64() + regular_file_bytes = reader.u64() + legal_file_count = reader.u16() + if legal_file_count == 0 or legal_file_count > 4096: + _fail(reader.artifact, ProvenanceReasonV1.INVALID_FIELD, "legal file count") + legal_files = tuple( + LegalFileV1.parse_from(reader) for _ in range(legal_file_count) + ) + integrity = _parse_integrity_policy(reader) + return cls( + role, + version, + archive_url, + archive_format, + archive_length, + archive_sha256, + tar_stream_length, + root_prefix, + regular_file_count, + regular_file_bytes, + legal_files, + integrity, + ) + + @cached_property + def identity(self) -> bytes: + return _identity(b"labcolors.proof-region.source-release-lock.v1\0", self.encode()) + + +@dataclass(frozen=True) +class ArbSourceLockV1: + sources: tuple[SourceReleaseLockV1, SourceReleaseLockV1, SourceReleaseLockV1] + + def __post_init__(self) -> None: + if type(self.sources) is not tuple or len(self.sources) != ARBITRARY_PRECISION_SOURCE_COUNT_V1: + _fail("arb-source-lock-v1", ProvenanceReasonV1.INVALID_FIELD, "source count") + if any(type(value) is not SourceReleaseLockV1 for value in self.sources): + _fail("arb-source-lock-v1", ProvenanceReasonV1.INVALID_FIELD, "source type") + if tuple(value.role for value in self.sources) != ( + SourceRoleV1.GMP, + SourceRoleV1.MPFR, + SourceRoleV1.FLINT_ARB, + ): + _fail("arb-source-lock-v1", ProvenanceReasonV1.NONCANONICAL_ORDER, "GMP, MPFR, FLINT") + if any( + not isinstance(value.integrity, DetachedSignaturePolicyV1) + for value in self.sources[:2] + ) or not isinstance( + self.sources[2].integrity, + GitContentRelationPolicyV1, + ): + _fail( + "arb-source-lock-v1", + ProvenanceReasonV1.INTEGRITY_KIND_MISMATCH, + "integrity policy", + ) + + def encode(self) -> bytes: + return ( + SOURCE_LOCK_MAGIC_V1 + + bytes((SOURCE_LOCK_RELEASE_V1, len(self.sources))) + + b"".join(source.encode() for source in self.sources) + ) + + @classmethod + def parse(cls, data: bytes) -> "ArbSourceLockV1": + reader = _Reader(data, "arb-source-lock-v1") + if reader.exact(len(SOURCE_LOCK_MAGIC_V1)) != SOURCE_LOCK_MAGIC_V1: + _fail(reader.artifact, ProvenanceReasonV1.BAD_MAGIC, "source lock magic") + if reader.u8() != SOURCE_LOCK_RELEASE_V1: + _fail(reader.artifact, ProvenanceReasonV1.UNKNOWN_RELEASE, "source lock release") + if reader.u8() != ARBITRARY_PRECISION_SOURCE_COUNT_V1: + _fail(reader.artifact, ProvenanceReasonV1.INVALID_FIELD, "source count") + result = cls(tuple(SourceReleaseLockV1.parse_from(reader) for _ in range(3))) + reader.finish() + if result.encode() != data: + _fail(reader.artifact, ProvenanceReasonV1.FOREIGN_BINDING, "re-encode drift") + return result + + @cached_property + def identity(self) -> bytes: + return _identity(SOURCE_LOCK_ID_LABEL_V1, self.encode()) + + +@dataclass(frozen=True) +class ArchiveFileV1: + path: str + mode: int + length: int + sha256: bytes + + +_SAFE_ARCHIVE_TOKEN = object() +_ADMITTED_ARB_SOURCES_TOKEN = object() + + +@dataclass(frozen=True, init=False) +class SafeSourceArchiveV1: + """Owned structural capability; it is neither origin nor build evidence. + + A materializer must consume archive_bytes from this value, never reopen a + pathname, and derive normalized directories from admitted regular files. + Empty archive directories intentionally carry no tree semantics. + """ + + source_lock_identity: bytes + archive_sha256: bytes + tree_identity: bytes + regular_file_count: int + regular_file_bytes: int + files: tuple[ArchiveFileV1, ...] + _archive_bytes: bytes = field(repr=False, compare=False) + + def __init__( + self, + source_lock_identity: bytes, + archive_sha256: bytes, + tree_identity: bytes, + regular_file_count: int, + regular_file_bytes: int, + files: tuple[ArchiveFileV1, ...], + archive_bytes: bytes, + *, + _token: object, + ) -> None: + if _token is not _SAFE_ARCHIVE_TOKEN: + raise TypeError("SafeSourceArchiveV1 is created only by archive admission") + object.__setattr__(self, "source_lock_identity", source_lock_identity) + object.__setattr__(self, "archive_sha256", archive_sha256) + object.__setattr__(self, "tree_identity", tree_identity) + object.__setattr__(self, "regular_file_count", regular_file_count) + object.__setattr__(self, "regular_file_bytes", regular_file_bytes) + object.__setattr__(self, "files", files) + object.__setattr__(self, "_archive_bytes", archive_bytes) + + @property + def archive_bytes(self) -> bytes: + """Return the immutable snapshot admitted by this capability.""" + + return self._archive_bytes + + +@dataclass(frozen=True, init=False) +class AdmittedArbSourcesV1: + """One ordered capability for the complete locked Arb dependency closure.""" + + source_lock_identity: bytes + sources: tuple[SafeSourceArchiveV1, SafeSourceArchiveV1, SafeSourceArchiveV1] + + def __init__( + self, + source_lock_identity: bytes, + sources: tuple[ + SafeSourceArchiveV1, + SafeSourceArchiveV1, + SafeSourceArchiveV1, + ], + *, + _token: object, + ) -> None: + if _token is not _ADMITTED_ARB_SOURCES_TOKEN: + raise TypeError("AdmittedArbSourcesV1 is created only by source admission") + _digest( + source_lock_identity, + "admitted-arb-sources-v1", + "source_lock_identity", + ) + if ( + type(sources) is not tuple + or len(sources) != ARBITRARY_PRECISION_SOURCE_COUNT_V1 + or any(type(source) is not SafeSourceArchiveV1 for source in sources) + ): + raise TypeError("invalid admitted Arb source tuple") + object.__setattr__(self, "source_lock_identity", source_lock_identity) + object.__setattr__(self, "sources", sources) + + @cached_property + def identity(self) -> bytes: + chunks = [self.source_lock_identity] + for ordinal, source in enumerate(self.sources): + chunks.extend( + ( + bytes((ordinal,)), + source.source_lock_identity, + source.archive_sha256, + source.tree_identity, + ) + ) + encoded = b"".join(chunks) + return _identity(ADMITTED_ARB_SOURCES_ID_LABEL_V1, encoded) + + +def _decompress_exact( + archive: bytes, + archive_format: ArchiveFormatV1, + expected_length: int, +) -> bytes: + try: + if archive_format is ArchiveFormatV1.TAR_GZIP: + decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS) + output = decompressor.decompress(archive, expected_length + 1) + if len(output) > expected_length: + _fail( + "source-archive-v1", + ProvenanceReasonV1.TAR_STREAM_LENGTH_MISMATCH, + "expanded beyond lock", + ) + while not decompressor.eof and decompressor.unconsumed_tail: + remaining = expected_length + 1 - len(output) + if remaining <= 0: + _fail( + "source-archive-v1", + ProvenanceReasonV1.TAR_STREAM_LENGTH_MISMATCH, + "expanded beyond lock", + ) + output += decompressor.decompress( + decompressor.unconsumed_tail, + remaining, + ) + if len(output) > expected_length: + break + if not decompressor.eof: + if len(output) > expected_length: + _fail( + "source-archive-v1", + ProvenanceReasonV1.TAR_STREAM_LENGTH_MISMATCH, + "expanded beyond lock", + ) + _fail( + "source-archive-v1", + ProvenanceReasonV1.DECOMPRESSION_FAILED, + "truncated gzip stream", + ) + if decompressor.unused_data: + _fail( + "source-archive-v1", + ProvenanceReasonV1.TRAILING_COMPRESSED_DATA, + "concatenated or trailing gzip data", + ) + else: + decompressor_xz = lzma.LZMADecompressor(format=lzma.FORMAT_XZ) + output = decompressor_xz.decompress(archive, max_length=expected_length + 1) + if len(output) > expected_length: + _fail( + "source-archive-v1", + ProvenanceReasonV1.TAR_STREAM_LENGTH_MISMATCH, + "expanded beyond lock", + ) + while not decompressor_xz.eof and not decompressor_xz.needs_input: + remaining = expected_length + 1 - len(output) + if remaining <= 0: + _fail( + "source-archive-v1", + ProvenanceReasonV1.TAR_STREAM_LENGTH_MISMATCH, + "expanded beyond lock", + ) + output += decompressor_xz.decompress( + b"", max_length=remaining + ) + if len(output) > expected_length: + break + if not decompressor_xz.eof: + if len(output) > expected_length: + _fail( + "source-archive-v1", + ProvenanceReasonV1.TAR_STREAM_LENGTH_MISMATCH, + "expanded beyond lock", + ) + _fail( + "source-archive-v1", + ProvenanceReasonV1.DECOMPRESSION_FAILED, + "truncated xz stream", + ) + if decompressor_xz.unused_data: + _fail( + "source-archive-v1", + ProvenanceReasonV1.TRAILING_COMPRESSED_DATA, + "concatenated or trailing xz data", + ) + except (zlib.error, lzma.LZMAError, EOFError): + _fail("source-archive-v1", ProvenanceReasonV1.DECOMPRESSION_FAILED, "invalid compressed stream") + if len(output) != expected_length: + _fail( + "source-archive-v1", + ProvenanceReasonV1.TAR_STREAM_LENGTH_MISMATCH, + "tar stream length", + ) + return output + + +def _tree_identity(files: tuple[ArchiveFileV1, ...]) -> bytes: + chunks = [len(files).to_bytes(8, "big")] + for item in files: + chunks.extend( + ( + _blob(item.path.encode("ascii")), + item.mode.to_bytes(4, "big"), + item.length.to_bytes(8, "big"), + item.sha256, + ) + ) + encoded = b"".join(chunks) + return _identity(SOURCE_TREE_ID_LABEL_V1, encoded) + + +def _scan_tar(expected: SourceReleaseLockV1, raw_tar: bytes) -> tuple[ArchiveFileV1, ...]: + files: list[ArchiveFileV1] = [] + seen: set[str] = set() + folded: set[str] = set() + directories: set[str] = set() + root = expected.root_prefix[:-1] + last_payload_end = 0 + admitted_file_bytes = 0 + try: + with tarfile.open(fileobj=io.BytesIO(raw_tar), mode="r:") as archive: + if archive.pax_headers: + _fail("source-archive-v1", ProvenanceReasonV1.NONCANONICAL_TAR, "global pax headers") + for member in archive: + if member.pax_headers: + _fail("source-archive-v1", ProvenanceReasonV1.NONCANONICAL_TAR, "member pax headers") + name = member.name + try: + name.encode("ascii") + except UnicodeEncodeError: + _fail("source-archive-v1", ProvenanceReasonV1.UNSAFE_PATH, "non-ASCII member") + if name.startswith("/"): + _fail("source-archive-v1", ProvenanceReasonV1.ABSOLUTE_PATH, name) + if "\\" in name or any(part in ("", ".", "..") for part in name.split("/")): + _fail("source-archive-v1", ProvenanceReasonV1.UNSAFE_PATH, name) + if name in seen: + _fail("source-archive-v1", ProvenanceReasonV1.DUPLICATE_PATH, name) + casefolded = name.lower() + if casefolded in folded: + _fail("source-archive-v1", ProvenanceReasonV1.CASE_COLLISION, name) + seen.add(name) + folded.add(casefolded) + last_payload_end = max( + last_payload_end, + member.offset_data + ((member.size + TAR_BLOCK_BYTES - 1) // TAR_BLOCK_BYTES) * TAR_BLOCK_BYTES, + ) + if member.issym() or member.islnk(): + _fail("source-archive-v1", ProvenanceReasonV1.UNSAFE_LINK, name) + if not (member.isdir() or member.isreg()): + _fail("source-archive-v1", ProvenanceReasonV1.UNSAFE_MEMBER_TYPE, name) + if member.isdir(): + if member.mode != ALLOWED_DIRECTORY_MODE_V1: + _fail("source-archive-v1", ProvenanceReasonV1.UNSAFE_MODE, name) + if name != root and not name.startswith(expected.root_prefix): + _fail("source-archive-v1", ProvenanceReasonV1.ROOT_MISMATCH, name) + parent = str(PurePosixPath(name).parent) + if name != root and parent not in directories: + _fail( + "source-archive-v1", + ProvenanceReasonV1.UNSAFE_PATH, + f"undeclared parent of {name}", + ) + directories.add(name) + continue + if not name.startswith(expected.root_prefix): + _fail("source-archive-v1", ProvenanceReasonV1.ROOT_MISMATCH, name) + relative = name[len(expected.root_prefix) :] + _relative_path(relative, "source-archive-v1", "member path") + parent = str(PurePosixPath(name).parent) + if parent not in directories: + _fail("source-archive-v1", ProvenanceReasonV1.UNSAFE_PATH, f"undeclared parent of {name}") + if member.mode not in ALLOWED_REGULAR_MODES_V1: + _fail("source-archive-v1", ProvenanceReasonV1.UNSAFE_MODE, name) + if len(files) >= expected.regular_file_count: + _fail("source-archive-v1", ProvenanceReasonV1.FILE_COUNT_MISMATCH, "too many files") + if member.size > expected.regular_file_bytes - admitted_file_bytes: + _fail("source-archive-v1", ProvenanceReasonV1.FILE_BYTES_MISMATCH, "declared bytes exceed lock") + stream = archive.extractfile(member) + if stream is None: + _fail("source-archive-v1", ProvenanceReasonV1.FILE_CONTENT_MISMATCH, name) + hasher = hashlib.sha256() + length = 0 + while True: + chunk = stream.read(READ_CHUNK_BYTES) + if not chunk: + break + length += len(chunk) + if length > member.size: + _fail("source-archive-v1", ProvenanceReasonV1.FILE_CONTENT_MISMATCH, name) + hasher.update(chunk) + if length != member.size: + _fail("source-archive-v1", ProvenanceReasonV1.FILE_CONTENT_MISMATCH, name) + files.append(ArchiveFileV1(relative, member.mode, length, hasher.digest())) + admitted_file_bytes += length + except tarfile.TarError: + _fail("source-archive-v1", ProvenanceReasonV1.NONCANONICAL_TAR, "invalid tar stream") + if root not in directories: + _fail("source-archive-v1", ProvenanceReasonV1.ROOT_MISMATCH, "missing root directory") + trailing = raw_tar[last_payload_end:] + if len(trailing) < TAR_END_MARKER_BYTES or any(trailing): + _fail("source-archive-v1", ProvenanceReasonV1.NONCANONICAL_TAR, "nonzero or missing tar terminator") + return tuple(sorted(files, key=lambda item: item.path)) + + +def _admit_source_archive_once( + expected: SourceReleaseLockV1, + archive: bytes, +) -> tuple[SafeSourceArchiveV1, bytes]: + + if type(expected) is not SourceReleaseLockV1: + raise TypeError("expected must be SourceReleaseLockV1") + if type(archive) is not bytes: + raise TypeError("archive must be owned bytes") + if len(archive) != expected.archive_length: + _fail("source-archive-v1", ProvenanceReasonV1.ARCHIVE_LENGTH_MISMATCH, "archive length") + archive_sha256 = hashlib.sha256(archive).digest() + if archive_sha256 != expected.archive_sha256: + _fail("source-archive-v1", ProvenanceReasonV1.ARCHIVE_DIGEST_MISMATCH, "archive digest") + raw_tar = _decompress_exact(archive, expected.archive_format, expected.tar_stream_length) + files = _scan_tar(expected, raw_tar) + if len(files) != expected.regular_file_count: + _fail("source-archive-v1", ProvenanceReasonV1.FILE_COUNT_MISMATCH, "regular file count") + total_bytes = sum(item.length for item in files) + if total_bytes != expected.regular_file_bytes: + _fail("source-archive-v1", ProvenanceReasonV1.FILE_BYTES_MISMATCH, "regular file bytes") + by_path = {item.path: item for item in files} + for legal_file in expected.legal_files: + actual = by_path.get(legal_file.path) + if ( + actual is None + or actual.length != legal_file.length + or actual.sha256 != legal_file.sha256 + ): + _fail( + "source-archive-v1", + ProvenanceReasonV1.LEGAL_FILES_MISMATCH, + legal_file.path, + ) + if isinstance(expected.integrity, GitContentRelationPolicyV1): + for path in expected.integrity.omitted_paths: + if path in by_path: + _fail( + "source-archive-v1", + ProvenanceReasonV1.CONTENT_RELATION_MISMATCH, + f"omitted path present: {path}", + ) + for release_only in expected.integrity.project_pinned_release_only_files: + actual = by_path.get(release_only.path) + if ( + actual is None + or actual.mode != release_only.mode + or actual.length != release_only.length + or actual.sha256 != release_only.sha256 + ): + _fail( + "source-archive-v1", + ProvenanceReasonV1.CONTENT_RELATION_MISMATCH, + release_only.path, + ) + tree_identity = _tree_identity(files) + admitted = SafeSourceArchiveV1( + expected.identity, + archive_sha256, + tree_identity, + len(files), + total_bytes, + files, + archive, + _token=_SAFE_ARCHIVE_TOKEN, + ) + return admitted, raw_tar + + +def admit_source_archive(expected: SourceReleaseLockV1, archive: bytes) -> SafeSourceArchiveV1: + """Hash then scan one locked archive; this establishes no origin trust.""" + + admitted, _raw_tar = _admit_source_archive_once(expected, archive) + return admitted + + +def replay_admitted_source_archive_v1( + expected: SourceReleaseLockV1, + admitted: SafeSourceArchiveV1, +) -> tuple[SafeSourceArchiveV1, bytes]: + """Re-admit owned bytes and return the raw tar from that exact pass. + + The caller cannot supply a second tar stream, so replay coordinates and + materialization bytes remain causally bound without decompressing twice. + """ + + if type(expected) is not SourceReleaseLockV1: + raise TypeError("expected must be SourceReleaseLockV1") + if type(admitted) is not SafeSourceArchiveV1: + raise TypeError("admitted must be SafeSourceArchiveV1") + return _admit_source_archive_once(expected, admitted.archive_bytes) + + +def admit_arb_sources( + expected: ArbSourceLockV1, + sources: tuple[ + SafeSourceArchiveV1, + SafeSourceArchiveV1, + SafeSourceArchiveV1, + ], +) -> AdmittedArbSourcesV1: + """Collapse three individually admitted archives into one ordered capability.""" + + if type(expected) is not ArbSourceLockV1: + raise TypeError("expected must be ArbSourceLockV1") + if ( + type(sources) is not tuple + or len(sources) != ARBITRARY_PRECISION_SOURCE_COUNT_V1 + or any(type(source) is not SafeSourceArchiveV1 for source in sources) + ): + raise TypeError("sources must be three SafeSourceArchiveV1 values") + for lock, source in zip(expected.sources, sources, strict=True): + if ( + source.source_lock_identity != lock.identity + or source.archive_sha256 != lock.archive_sha256 + or source.regular_file_count != lock.regular_file_count + or source.regular_file_bytes != lock.regular_file_bytes + ): + _fail( + "admitted-arb-sources-v1", + ProvenanceReasonV1.FOREIGN_BINDING, + "source capability does not match ordered lock", + ) + return AdmittedArbSourcesV1( + expected.identity, + sources, + _token=_ADMITTED_ARB_SOURCES_TOKEN, + ) + + +def _legal_file(path: str, length: int, digest_hex: str) -> LegalFileV1: + return LegalFileV1(path, length, bytes.fromhex(digest_hex)) + + +def arb_source_lock_v1() -> ArbSourceLockV1: + """Return the exact published source declarations for the first Arb lane.""" + + gmp = SourceReleaseLockV1( + SourceRoleV1.GMP, + "6.3.0", + "https://ftp.gnu.org/gnu/gmp/gmp-6.3.0.tar.xz", + ArchiveFormatV1.TAR_XZ, + 2_094_196, + bytes.fromhex("a3c2b80201b89e68616f4ad30bc66aee4927c3ce50e33929ca819d5c43538898"), + 18_759_680, + "gmp-6.3.0/", + 2_156, + 16_998_222, + ( + _legal_file("COPYING", 35_147, "8ceb4b9ee5adedde47b31e975c1d90c73ad27b6b165a1dcd80c7c545eb65b903"), + _legal_file("COPYING.LESSERv3", 7_639, "a853c2ffec17057872340eee242ae4d96cbf2b520ae27d903e1b2fef1a5f9d1c"), + _legal_file("COPYINGv2", 18_092, "8177f97513213526df2cf6184d8ff986c675afb514d4e68a404010521b880643"), + _legal_file("COPYINGv3", 35_150, "e6037104443f9a7829b2aa7c5370d0789a7bda3ca65a0b904cdc0c2e285d9195"), + _legal_file("README", 4_051, "5e9f9325fd702bc4bcda27d7a78fea88a2a09fa39b4b15ac7b9b205e0863dc7e"), + ), + DetachedSignaturePolicyV1( + "https://ftp.gnu.org/gnu/gmp/gmp-6.3.0.tar.xz.sig", + 374, + bytes.fromhex("94def8c1a731854de684689126046ec93589147abd4cd0025f12d741d323aa82"), + bytes.fromhex("928ac84aa0e2134bbb335cd439110dc3f9b967eb04caff4a44dd5d04a3f13474"), + bytes.fromhex("343c2ff0fbee5ec2edbef399f3599ff828c67298"), + ), + ) + mpfr = SourceReleaseLockV1( + SourceRoleV1.MPFR, + "4.2.2", + "https://www.mpfr.org/mpfr-4.2.2/mpfr-4.2.2.tar.xz", + ArchiveFormatV1.TAR_XZ, + 1_505_596, + bytes.fromhex("b67ba0383ef7e8a8563734e2e889ef5ec3c3b898a01d00fa0a6869ad81c6ce01"), + 10_045_440, + "mpfr-4.2.2/", + 572, + 9_590_620, + ( + _legal_file("COPYING", 35_149, "3972dc9744f6499f0f9b2dbf76696f2ae7ad8af9b23dde66d6af86c9dfb36986"), + _legal_file("COPYING.LESSER", 7_652, "e3a994d82e644b03a792a930f574002658412f62407f5fee083f2555c5f23118"), + _legal_file("README", 3_333, "74e733d2cfa1a6f4e6530326ed460f13ac9e4a5d79bb0f682ab67db2c9dc4d5b"), + ), + DetachedSignaturePolicyV1( + "https://www.mpfr.org/mpfr-4.2.2/mpfr-4.2.2.tar.xz.asc", + 228, + bytes.fromhex("c6264c9a3652bc40775205ce90e7c96cea5058629e2e68f9eede5d8213f23ee6"), + bytes.fromhex("3fe00f68bbf3888ae185b950d4db0f708dd01b6159cb03dec77296f9045b6372"), + bytes.fromhex("a534be3f83e241d918280aeb5831d11a0d4db02a"), + ), + ) + omitted = ( + ".gitattributes", + ".github/ISSUE_TEMPLATE/bug_report.md", + ".github/ISSUE_TEMPLATE/feature_request.md", + ".github/PULL_REQUEST_TEMPLATE/pull_request_template.md", + ".github/codecov.yml", + ".github/workflows/CI.yml", + ".github/workflows/docs.yml", + ".github/workflows/push_CI.yml", + ".github/workflows/release.yml", + ".gitignore", + "dev/bench.py", + "dev/check_examples.sh", + "dev/check_prototypes", + "dev/conway/convert_cp_to_new_form.jl", + "dev/conway/notes.c", + "dev/find_gmp_mpfr.jl", + "dev/gen_mul_basecase.jl", + "dev/gen_mul_basecase.py", + "dev/gen_mulhigh_basecase.jl", + "dev/make_dist.sh", + ) + project_pinned_release_only_files = ( + ProjectPinnedReleaseOnlyFileV1( + "config/install-sh", + 0o700, + 15_358, + bytes.fromhex("3d7488bebd0cfc9b5c440c55d5b44f1c6e2e3d3e19894821bae4a27f9307f1d2"), + ), + ProjectPinnedReleaseOnlyFileV1( + "config/ltmain.sh", + 0o755, + 333_053, + bytes.fromhex("579a1445e6a9a8b0809a44aa9f908387d4a43a2a440c9b84ea979f2b4f17816c"), + ), + ProjectPinnedReleaseOnlyFileV1( + "configure", + 0o755, + 731_646, + bytes.fromhex("43192d2f63812610726d943ada13bfc25864c39a8555314395d2d459d1502f45"), + ), + ProjectPinnedReleaseOnlyFileV1( + "src/config.h.in", + 0o644, + 6_645, + bytes.fromhex("af5b88c82a1549585b43a5dc856f3325d3513f423da0880f5459d913a25f9455"), + ), + ) + flint = SourceReleaseLockV1( + SourceRoleV1.FLINT_ARB, + "3.6.0", + "https://github.com/flintlib/flint/releases/download/v3.6.0/flint-3.6.0.tar.gz", + ArchiveFormatV1.TAR_GZIP, + 9_313_139, + bytes.fromhex("b95e2c7792f5eea4a1c8d2d42c4098434756832e57a094b295eb5dfdc9b4c36b"), + 56_811_520, + "flint-3.6.0/", + 10_112, + 48_758_775, + ( + _legal_file("COPYING", 35_149, "3972dc9744f6499f0f9b2dbf76696f2ae7ad8af9b23dde66d6af86c9dfb36986"), + _legal_file("COPYING.LESSER", 7_652, "e3a994d82e644b03a792a930f574002658412f62407f5fee083f2555c5f23118"), + _legal_file("README.md", 3_008, "1a1c629fe32957b0bdf197c6048a83a987e8d28793234aff6feec5e1dcf7633f"), + ), + GitContentRelationPolicyV1( + "https://github.com/flintlib/flint.git", + "v3.6.0", + bytes.fromhex("8d5454b96761fafe4d5a9da76a369a602f500f49"), + bytes.fromhex("18d57417a96227b27dd5336881403dee6fdc851b"), + 10_108, + omitted, + project_pinned_release_only_files, + ), + ) + return ArbSourceLockV1((gmp, mpfr, flint)) diff --git a/proof/region/v1/region_proof_protocol.py b/proof/region/v1/region_proof_protocol.py index f11cc2ff..48db70fe 100644 --- a/proof/region/v1/region_proof_protocol.py +++ b/proof/region/v1/region_proof_protocol.py @@ -47,7 +47,7 @@ DOMAIN_MAGIC_V1 = b"LCDOM1\0\0" POLICY_MAGIC_V1 = b"LCPOL1\0\0" JOB_MAGIC_V1 = b"LCJOB1\0\0" -MANIFEST_MAGIC_V1 = b"LCMAN1\0\0" +MANIFEST_MAGIC_V2 = b"LCMAN2\0\0" TRANSCRIPT_MAGIC_V1 = b"LCTRN1\0\0" RUN_CLAIM_MAGIC_V1 = b"LCRUN1\0\0" PROVENANCE_CLAIM_MAGIC_V1 = b"LCPRV1\0\0" @@ -56,7 +56,7 @@ DOMAIN_ID_LABEL_V1 = b"labcolors.proof-region.domain.v1\0" POLICY_ID_LABEL_V1 = b"labcolors.proof-region.policy.v1\0" JOB_ID_LABEL_V1 = b"labcolors.proof-region.job.v1\0" -MANIFEST_ID_LABEL_V1 = b"labcolors.proof-region.comparator-manifest.v1\0" +MANIFEST_ID_LABEL_V2 = b"labcolors.proof-region.comparator-manifest.v2\0" TRANSCRIPT_ID_LABEL_V1 = b"labcolors.proof-region.transcript.v1\0" RUN_CLAIM_ID_LABEL_V1 = b"labcolors.proof-region.run-claim.v1\0" PROVENANCE_CLAIM_ID_LABEL_V1 = b"labcolors.proof-region.evaluator-provenance-claim.v1\0" @@ -714,30 +714,34 @@ def identity(self) -> bytes: @dataclass(frozen=True) -class ComparatorManifestV1: +class ComparatorManifestV2: kind: ComparatorKindV1 engine_release: bytes upstream_source: bytes - arithmetic_closure: bytes + arithmetic_input_set: bytes wrapper_source: bytes evaluator_source: bytes build_identity: bytes operation_allowlist: bytes - test_receipt: bytes - license_closure: bytes + test_observation: bytes + legal_file_set: bytes exclusions: bytes def __post_init__(self) -> None: if type(self.kind) is not ComparatorKindV1: - _fail("comparator-manifest-v1", 0, ProtocolReasonV1.UNKNOWN_RELEASE, "unknown comparator kind") - for field in fields(self): - if field.name != "kind": - _require_digest(getattr(self, field.name), "comparator-manifest-v1", field.name) + _fail("comparator-manifest-v2", 0, ProtocolReasonV1.UNKNOWN_RELEASE, "unknown comparator kind") + for manifest_field in fields(self): + if manifest_field.name != "kind": + _require_digest( + getattr(self, manifest_field.name), + "comparator-manifest-v2", + manifest_field.name, + ) @classmethod - def parse(cls, data: bytes) -> "ComparatorManifestV1": - reader = _Reader(data, "comparator-manifest-v1") - reader.magic(MANIFEST_MAGIC_V1) + def parse(cls, data: bytes) -> "ComparatorManifestV2": + reader = _Reader(data, "comparator-manifest-v2") + reader.magic(MANIFEST_MAGIC_V2) kind_offset = reader.offset try: kind = ComparatorKindV1(reader.u8()) @@ -751,33 +755,35 @@ def parse(cls, data: bytes) -> "ComparatorManifestV1": return result def encode(self) -> bytes: - return MANIFEST_MAGIC_V1 + bytes((int(self.kind),)) + b"".join( - getattr(self, field.name) for field in fields(self) if field.name != "kind" + return MANIFEST_MAGIC_V2 + bytes((int(self.kind),)) + b"".join( + getattr(self, manifest_field.name) + for manifest_field in fields(self) + if manifest_field.name != "kind" ) @cached_property def identity(self) -> bytes: - return _identity(MANIFEST_ID_LABEL_V1, self.encode()) + return _identity(MANIFEST_ID_LABEL_V2, self.encode()) @dataclass(frozen=True, init=False) -class ContentResolvedComparatorManifestV1: - manifest: ComparatorManifestV1 +class ContentResolvedComparatorManifestV2: + manifest: ComparatorManifestV2 def __new__(cls): - raise TypeError("use ContentResolvedComparatorManifestV1.admit") + raise TypeError("use ContentResolvedComparatorManifestV2.admit") @classmethod def admit( cls, - manifest: ComparatorManifestV1, + manifest: ComparatorManifestV2, resolve_content_address: Callable[[bytes], bytes | Iterable[bytes] | None], - ) -> "ContentResolvedComparatorManifestV1": + ) -> "ContentResolvedComparatorManifestV2": # A digest declaration alone is not source binding. This structural # transition only re-hashes caller-provided bytes; a future controlled # replay must establish where those bytes came from. - if type(manifest) is not ComparatorManifestV1: + if type(manifest) is not ComparatorManifestV2: _fail( - "comparator-manifest-v1", + "comparator-manifest-v2", 0, ProtocolReasonV1.INVALID_MANIFEST, "content resolution requires a canonical manifest", @@ -789,7 +795,7 @@ def admit( content = resolve_content_address(coordinate) if content is None: _fail( - "comparator-manifest-v1", + "comparator-manifest-v2", 0, ProtocolReasonV1.INVALID_MANIFEST, f"unresolved content address: {field.name}", @@ -801,7 +807,7 @@ def admit( chunks = iter(content) except TypeError: _fail( - "comparator-manifest-v1", + "comparator-manifest-v2", 0, ProtocolReasonV1.INVALID_MANIFEST, f"content resolver did not return bytes: {field.name}", @@ -810,7 +816,7 @@ def admit( for chunk in chunks: if type(chunk) is not bytes: _fail( - "comparator-manifest-v1", + "comparator-manifest-v2", 0, ProtocolReasonV1.INVALID_MANIFEST, f"non-byte content chunk: {field.name}", @@ -818,7 +824,7 @@ def admit( replay.update(chunk) if replay.digest() != coordinate: _fail( - "comparator-manifest-v1", + "comparator-manifest-v2", 0, ProtocolReasonV1.DIGEST_MISMATCH, f"content digest mismatch: {field.name}", @@ -1249,7 +1255,7 @@ def iter_witnesses(self) -> Iterator[WitnessV1]: cursor = end -def _validate_witness_alignment( +def validate_witness_alignment_v1( domain: ReducedDomainManifestV1, decision_bits: bytes, point_count: int, @@ -1333,7 +1339,7 @@ def __post_init__(self) -> None: def from_decisions( cls, job: ProofJobV1, - comparator: ContentResolvedComparatorManifestV1, + comparator: ContentResolvedComparatorManifestV2, decisions: Iterable[DecisionV1], witnesses: Iterable[WitnessV1], accounting_digest: bytes, @@ -1357,7 +1363,7 @@ def from_decisions( accounting_digest, witness_store, ) - _validate_witness_alignment( + validate_witness_alignment_v1( job.domain, result.decision_bits, result.point_count, @@ -1496,7 +1502,7 @@ def __post_init__(self) -> None: def for_transcript( cls, job: ProofJobV1, - comparator: ContentResolvedComparatorManifestV1, + comparator: ContentResolvedComparatorManifestV2, transcript: DecisionTranscriptV1, binary_identity: bytes, invocation_identity: bytes, @@ -1667,7 +1673,7 @@ def identity(self) -> bytes: def _admit_transcript( job: ProofJobV1, - comparator: ContentResolvedComparatorManifestV1, + comparator: ContentResolvedComparatorManifestV2, transcript: DecisionTranscriptV1, run: RunClaimV1, *, @@ -1686,7 +1692,7 @@ def _admit_transcript( or run.transcript_identity != transcript_identity ): _fail("dual-admission-v1", 0, ProtocolReasonV1.FOREIGN_BINDING, "foreign transcript/run coordinate") - _validate_witness_alignment( + validate_witness_alignment_v1( job.domain, transcript.decision_bits, transcript.point_count, @@ -1697,10 +1703,10 @@ def _admit_transcript( def compare_dual_transcripts( job: ProofJobV1, - first_manifest: ContentResolvedComparatorManifestV1, + first_manifest: ContentResolvedComparatorManifestV2, first_transcript: DecisionTranscriptV1, first_run: RunClaimV1, - second_manifest: ContentResolvedComparatorManifestV1, + second_manifest: ContentResolvedComparatorManifestV2, second_transcript: DecisionTranscriptV1, second_run: RunClaimV1, ) -> DualComparisonCandidateV1: @@ -1718,10 +1724,10 @@ def compare_dual_transcripts( "dual admission requires canonical job, transcripts and runs", ) if ( - type(first_manifest) is not ContentResolvedComparatorManifestV1 - or type(second_manifest) is not ContentResolvedComparatorManifestV1 - or type(first_manifest.manifest) is not ComparatorManifestV1 - or type(second_manifest.manifest) is not ComparatorManifestV1 + type(first_manifest) is not ContentResolvedComparatorManifestV2 + or type(second_manifest) is not ContentResolvedComparatorManifestV2 + or type(first_manifest.manifest) is not ComparatorManifestV2 + or type(second_manifest.manifest) is not ComparatorManifestV2 ): _fail( "dual-admission-v1", diff --git a/proof/region/v1/tests/test_region_proof_protocol.py b/proof/region/v1/tests/test_region_proof_protocol.py index 9acf0f00..b1e6d5a5 100644 --- a/proof/region/v1/tests/test_region_proof_protocol.py +++ b/proof/region/v1/tests/test_region_proof_protocol.py @@ -33,7 +33,7 @@ FORMULA_RELEASE_DOMAIN_V1, ComparatorBudgetV1, ComparatorKindV1, - ComparatorManifestV1, + ComparatorManifestV2, BoundaryUnprovenWitnessV1, ContextualRegionDefinitionV1, DecisionTranscriptV1, @@ -49,7 +49,7 @@ ResourceLimitWitnessV1, RunClaimV1, WitnessStoreV1, - ContentResolvedComparatorManifestV1, + ContentResolvedComparatorManifestV2, compare_dual_transcripts, encode_contextual_definition_fields_v1, ) @@ -74,16 +74,16 @@ "6e493856d3c81f0d5b12bf1221985c66210ae98c8b6c79c7c5b4aabf243c0116" ) MANIFEST_IDENTITY = bytes.fromhex( - "77373d7025d4a673db27e7ce2ca45e61f9f191f7e017f97731602997430b5739" + "805c3710b9b38189f4b9c0bb69aaf429c944637a4ee38d1ffa56ee2d72ec09d9" ) TRANSCRIPT_IDENTITY = bytes.fromhex( - "d75c7f5d1c8176fdef78cca226e42ecd0cd61bab69b636fe4397e6a763af8582" + "8de25059cf372364da4d6cea05f9a45def3a3a9edfd385443cc31e7d82b59136" ) RUN_CLAIM_IDENTITY = bytes.fromhex( - "3ffae955c59e0cffa53cfa560713fff64f4c63f7ae103eae67c62a068e124b72" + "2c2e2ea8f0737e77a456306ad15332ab1dda609b872260c6bf76229c04b1ad9b" ) COMPARISON_IDENTITY = bytes.fromhex( - "94be6dfc28c1bcc98b703f9fda6e7231984a129e22db554655d4026dae5c4ba0" + "45ee817424540eaed060fcbfc7a43aebb1e0d484759f8a88803c2ed1a2648b9f" ) @@ -102,9 +102,9 @@ def digest(label: int) -> bytes: def admit_manifest( - value: ComparatorManifestV1, -) -> ContentResolvedComparatorManifestV1: - return ContentResolvedComparatorManifestV1.admit( + value: ComparatorManifestV2, +) -> ContentResolvedComparatorManifestV2: + return ContentResolvedComparatorManifestV2.admit( value, SYNTHETIC_CONTENT.get, ) @@ -138,19 +138,19 @@ def fixture_policy() -> ProofPolicyV1: ) -def manifest(kind: ComparatorKindV1, seed: int) -> ContentResolvedComparatorManifestV1: +def manifest(kind: ComparatorKindV1, seed: int) -> ContentResolvedComparatorManifestV2: return admit_manifest( - ComparatorManifestV1( + ComparatorManifestV2( kind=kind, engine_release=digest(seed), upstream_source=digest(seed + 1), - arithmetic_closure=digest(seed + 2), + arithmetic_input_set=digest(seed + 2), wrapper_source=digest(seed + 3), evaluator_source=digest(seed + 4), build_identity=digest(seed + 5), operation_allowlist=digest(seed + 6), - test_receipt=digest(seed + 7), - license_closure=digest(seed + 8), + test_observation=digest(seed + 7), + legal_file_set=digest(seed + 8), exclusions=digest(seed + 9), ) ) @@ -671,24 +671,72 @@ class ManifestTranscriptComparisonTests(unittest.TestCase): def test_protocol_slice_cannot_name_structural_agreement_a_proof(self) -> None: self.assertFalse(hasattr(protocol, "DualProofReceiptV1")) + def test_manifest_v1_surface_is_hard_deleted(self) -> None: + for name in ( + "MANIFEST_MAGIC_V1", + "MANIFEST_ID_LABEL_V1", + "ComparatorManifestV1", + "ContentResolvedComparatorManifestV1", + ): + with self.subTest(name=name): + self.assertFalse(hasattr(protocol, name)) + + def test_manifest_v2_rejects_the_historical_v1_wire_domain(self) -> None: + coordinates = tuple(digest(index) for index in range(10, 20)) + legacy_wire = ( + b"LCMAN1\0\0" + + bytes((int(ComparatorKindV1.ARB),)) + + b"".join(coordinates) + ) + + expect_reason( + self, + ProtocolReasonV1.BAD_MAGIC, + lambda: protocol.ComparatorManifestV2.parse(legacy_wire), + ) + + def test_manifest_v2_has_a_distinct_wire_and_identity_domain(self) -> None: + current = protocol.ComparatorManifestV2( + ComparatorKindV1.ARB, + *(digest(index) for index in range(10, 20)), + ) + encoded = current.encode() + + self.assertEqual(encoded[:8], b"LCMAN2\0\0") + self.assertEqual( + current.identity, + hashlib.sha256( + b"labcolors.proof-region.comparator-manifest.v2\0" + + len(encoded).to_bytes(8, "big") + + encoded + ).digest(), + ) + legacy_wire = b"LCMAN1\0\0" + encoded[8:] + legacy_identity = hashlib.sha256( + b"labcolors.proof-region.comparator-manifest.v1\0" + + len(legacy_wire).to_bytes(8, "big") + + legacy_wire + ).digest() + self.assertNotEqual(current.identity, legacy_identity) + def test_manifest_is_content_resolved_and_each_field_changes_identity(self) -> None: base = manifest(ComparatorKindV1.ARB, 10) self.assertEqual( admit_manifest( - ComparatorManifestV1.parse(base.manifest.encode()) + ComparatorManifestV2.parse(base.manifest.encode()) ).identity, base.identity, ) for field in ( "engine_release", "upstream_source", - "arithmetic_closure", + "arithmetic_input_set", "wrapper_source", "evaluator_source", "build_identity", "operation_allowlist", - "test_receipt", - "license_closure", + "test_observation", + "legal_file_set", "exclusions", ): changed = admit_manifest( @@ -702,7 +750,7 @@ def test_manifest_is_content_resolved_and_each_field_changes_identity(self) -> N expect_reason( self, ProtocolReasonV1.INVALID_MANIFEST, - lambda coordinate=coordinate: ContentResolvedComparatorManifestV1.admit( + lambda coordinate=coordinate: ContentResolvedComparatorManifestV2.admit( base.manifest, lambda current: ( None @@ -714,7 +762,7 @@ def test_manifest_is_content_resolved_and_each_field_changes_identity(self) -> N expect_reason( self, ProtocolReasonV1.DIGEST_MISMATCH, - lambda coordinate=coordinate: ContentResolvedComparatorManifestV1.admit( + lambda coordinate=coordinate: ContentResolvedComparatorManifestV2.admit( base.manifest, lambda current: ( b"wrong" @@ -727,7 +775,7 @@ def test_manifest_is_content_resolved_and_each_field_changes_identity(self) -> N expect_reason( self, ProtocolReasonV1.UNKNOWN_RELEASE, - lambda: ComparatorManifestV1( + lambda: ComparatorManifestV2( 3, # type: ignore[arg-type] *(digest(index) for index in range(300, 310)), ), @@ -743,13 +791,13 @@ def test_manifest_is_content_resolved_and_each_field_changes_identity(self) -> N expect_reason( self, ProtocolReasonV1.INVALID_MANIFEST, - lambda: ContentResolvedComparatorManifestV1.admit( # type: ignore[arg-type] + lambda: ContentResolvedComparatorManifestV2.admit( # type: ignore[arg-type] lookalike, SYNTHETIC_CONTENT.get, ), ) with self.assertRaises(TypeError): - ContentResolvedComparatorManifestV1() # type: ignore[call-arg] + ContentResolvedComparatorManifestV2() # type: ignore[call-arg] class ForeignBytes(bytes): pass @@ -765,7 +813,7 @@ class ForeignBytes(bytes): expect_reason( self, ProtocolReasonV1.INVALID_MANIFEST, - lambda invalid_content=invalid_content: ContentResolvedComparatorManifestV1.admit( + lambda invalid_content=invalid_content: ContentResolvedComparatorManifestV2.admit( base.manifest, lambda _coordinate: invalid_content, # type: ignore[return-value] ), @@ -1242,25 +1290,25 @@ def test_synthetic_resolved_transcripts_produce_only_structural_comparison(self) ( arb.manifest, 329, - "6323e41b60305ef9cf19b4ad65450779079adc87968ed1fdf8a70952f7b39166", + "884625d5983131234d0570f90b482e0e9de2801b773f7f03e34eb2138b104c30", MANIFEST_IDENTITY, ), ( ta, 328, - "ae55a7e363a6fdd2f8cb1455ecc45b4ef937538421c017014e033e9a955c3402", + "b1bc17383b99683302d9156ef1b784a0f094e6eba4e99a346a0a3331c47db3cb", TRANSCRIPT_IDENTITY, ), ( ra, 200, - "5af9da154b5c2e6f5dbdf88b538324fe809366dab712227f95a67657046d06b2", + "4a23db54ac34d2117326d645b17fae098a49a8ec54ea1cc3955d5a1328c7053c", RUN_CLAIM_IDENTITY, ), ( candidate, 368, - "f27fb4ad6fc8e14d16f815b394f67e181d29d02099c2d640f5dec07e38e63f3d", + "017dd72a3dcf001acdd267f91a79a32926da8351874a534ce968c0cf016c0026", COMPARISON_IDENTITY, ), ): diff --git a/proof/region/v1/tests/test_source_lock.py b/proof/region/v1/tests/test_source_lock.py new file mode 100644 index 00000000..5db241b9 --- /dev/null +++ b/proof/region/v1/tests/test_source_lock.py @@ -0,0 +1,598 @@ +#!/usr/bin/env python3 +"""Hostile source-lock and archive-admission tests for proof tooling V1.""" + +from __future__ import annotations + +import gzip +import hashlib +import io +import lzma +import sys +import tarfile +import unittest +from dataclasses import replace +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import provenance # noqa: E402 +from provenance import ( # noqa: E402 + AdmittedArbSourcesV1, + ArchiveFormatV1, + DetachedSignaturePolicyV1, + GitContentRelationPolicyV1, + LegalFileV1, + ProjectPinnedReleaseOnlyFileV1, + ProvenanceErrorV1, + ProvenanceReasonV1, + SourceReleaseLockV1, + SourceRoleV1, + admit_source_archive, + admit_arb_sources, + arb_source_lock_v1, +) + + +def sha256(value: bytes) -> bytes: + return hashlib.sha256(value).digest() + + +def tar_gz( + entries: tuple[tuple[str, bytes | None, bytes | None], ...], +) -> bytes: + """Build deterministic hostile fixtures; linkname is the third item.""" + + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode="w", format=tarfile.USTAR_FORMAT) as archive: + for name, body, linkname in entries: + member = tarfile.TarInfo(name) + member.mtime = 0 + member.uid = 0 + member.gid = 0 + member.uname = "" + member.gname = "" + if linkname is not None: + member.type = tarfile.SYMTYPE + member.linkname = linkname.decode("ascii") + member.mode = 0o777 + archive.addfile(member) + elif body is None: + member.type = tarfile.DIRTYPE + member.mode = 0o755 + archive.addfile(member) + else: + member.type = tarfile.REGTYPE + member.mode = 0o644 + member.size = len(body) + archive.addfile(member, io.BytesIO(body)) + return gzip.compress(raw.getvalue(), compresslevel=9, mtime=0) + + +def fixture_lock( + archive: bytes, + *, + root: str = "fixture-1/", + file_count: int = 2, + unpacked_bytes: int = 11, + tar_stream_bytes: int | None = None, + archive_format: ArchiveFormatV1 = ArchiveFormatV1.TAR_GZIP, +) -> SourceReleaseLockV1: + if tar_stream_bytes is None: + tar_stream_bytes = len( + gzip.decompress(archive) + if archive_format is ArchiveFormatV1.TAR_GZIP + else lzma.decompress(archive) + ) + return SourceReleaseLockV1( + role=SourceRoleV1.GMP, + version="1", + archive_url="https://example.invalid/fixture-1.tar.gz", + archive_format=archive_format, + archive_length=len(archive), + archive_sha256=sha256(archive), + tar_stream_length=tar_stream_bytes, + root_prefix=root, + regular_file_count=file_count, + regular_file_bytes=unpacked_bytes, + legal_files=(LegalFileV1("LICENSE", 7, sha256(b"license")),), + integrity=DetachedSignaturePolicyV1( + signature_url="https://example.invalid/fixture-1.tar.gz.sig", + signature_length=3, + signature_sha256=sha256(b"sig"), + public_key_packets_sha256=sha256(b"packets"), + signer_fingerprint=bytes.fromhex( + "00112233445566778899aabbccddeeff00112233" + ), + ), + ) + + +GOOD_ARCHIVE = tar_gz( + ( + ("fixture-1/", None, None), + ("fixture-1/LICENSE", b"license", None), + ("fixture-1/value", b"data", None), + ) +) + + +class ArbSourceLockTests(unittest.TestCase): + def test_old_overclaiming_vocabulary_is_not_public(self) -> None: + for name in ( + "GeneratedFileV1", + "GitReleasePolicyV1", + "LicenseFileV1", + "OriginKindV1", + "OriginPolicyV1", + ): + with self.subTest(name=name): + self.assertFalse(hasattr(provenance, name)) + self.assertFalse( + hasattr(ProvenanceReasonV1, "LICENSE_CLOSURE_MISMATCH") + ) + self.assertFalse( + hasattr(ProvenanceReasonV1, "RELEASE_RELATION_MISMATCH") + ) + self.assertFalse( + hasattr(provenance.IntegrityKindV1, "GIT_RELEASE_RELATION") + ) + self.assertFalse(hasattr(ProvenanceReasonV1, "ORIGIN_KIND_MISMATCH")) + self.assertFalse(hasattr(provenance, "_parse_origin_policy")) + self.assertNotIn("origin", SourceReleaseLockV1.__dataclass_fields__) + self.assertIn("integrity", SourceReleaseLockV1.__dataclass_fields__) + + def test_exact_primary_coordinates_are_canonical_and_round_trip(self) -> None: + lock = arb_source_lock_v1() + self.assertEqual(tuple(item.role for item in lock.sources), ( + SourceRoleV1.GMP, + SourceRoleV1.MPFR, + SourceRoleV1.FLINT_ARB, + )) + + gmp, mpfr, flint = lock.sources + self.assertEqual(gmp.version, "6.3.0") + self.assertEqual(gmp.archive_length, 2_094_196) + self.assertEqual( + gmp.archive_sha256.hex(), + "a3c2b80201b89e68616f4ad30bc66aee4927c3ce50e33929ca819d5c43538898", + ) + self.assertIsInstance(gmp.integrity, DetachedSignaturePolicyV1) + self.assertEqual( + gmp.integrity.signer_fingerprint.hex(), + "343c2ff0fbee5ec2edbef399f3599ff828c67298", + ) + self.assertEqual( + gmp.integrity.public_key_packets_sha256.hex(), + "928ac84aa0e2134bbb335cd439110dc3f9b967eb04caff4a44dd5d04a3f13474", + ) + + self.assertEqual(mpfr.version, "4.2.2") + self.assertEqual( + mpfr.archive_url, + "https://www.mpfr.org/mpfr-4.2.2/mpfr-4.2.2.tar.xz", + ) + self.assertEqual(mpfr.archive_length, 1_505_596) + self.assertEqual( + mpfr.archive_sha256.hex(), + "b67ba0383ef7e8a8563734e2e889ef5ec3c3b898a01d00fa0a6869ad81c6ce01", + ) + self.assertIsInstance(mpfr.integrity, DetachedSignaturePolicyV1) + self.assertEqual( + mpfr.integrity.signer_fingerprint.hex(), + "a534be3f83e241d918280aeb5831d11a0d4db02a", + ) + self.assertEqual( + mpfr.integrity.public_key_packets_sha256.hex(), + "3fe00f68bbf3888ae185b950d4db0f708dd01b6159cb03dec77296f9045b6372", + ) + + self.assertEqual(flint.version, "3.6.0") + self.assertEqual(flint.archive_length, 9_313_139) + self.assertEqual( + flint.archive_sha256.hex(), + "b95e2c7792f5eea4a1c8d2d42c4098434756832e57a094b295eb5dfdc9b4c36b", + ) + self.assertIsInstance(flint.integrity, GitContentRelationPolicyV1) + self.assertEqual( + flint.integrity.commit.hex(), + "8d5454b96761fafe4d5a9da76a369a602f500f49", + ) + self.assertEqual( + flint.integrity.tree.hex(), + "18d57417a96227b27dd5336881403dee6fdc851b", + ) + self.assertEqual(flint.integrity.common_file_count, 10_108) + self.assertEqual(len(flint.integrity.omitted_paths), 20) + self.assertEqual( + len(flint.integrity.project_pinned_release_only_files), + 4, + ) + + encoded = lock.encode() + self.assertEqual(len(encoded), 2_286) + self.assertEqual( + lock.identity.hex(), + "a4948c57ed0f9bb066a285b17d7990415cad22ff8d03b5f91900b73da5d2b8cc", + ) + self.assertEqual(type(lock).parse(encoded).encode(), encoded) + self.assertEqual(type(lock).parse(encoded).identity, lock.identity) + + def test_every_expected_coordinate_is_identity_bound(self) -> None: + lock = arb_source_lock_v1() + seen: set[bytes] = set() + for index, source in enumerate(lock.sources): + if isinstance(source.integrity, GitContentRelationPolicyV1): + count_mutation = replace( + source, + regular_file_count=source.regular_file_count + 1, + integrity=replace( + source.integrity, + common_file_count=source.integrity.common_file_count + 1, + ), + ) + else: + count_mutation = replace( + source, + regular_file_count=source.regular_file_count + 1, + ) + source_mutations = ( + replace(source, version=source.version + "x"), + replace(source, archive_url=source.archive_url + ".invalid"), + replace(source, archive_length=source.archive_length + 1), + replace(source, archive_sha256=sha256(source.archive_sha256)), + replace(source, tar_stream_length=source.tar_stream_length + 512), + replace(source, root_prefix="x-" + source.root_prefix), + count_mutation, + replace(source, regular_file_bytes=source.regular_file_bytes + 1), + replace( + source, + legal_files=( + replace( + source.legal_files[0], + length=source.legal_files[0].length + 1, + ), + ) + + source.legal_files[1:], + ), + ) + if isinstance(source.integrity, DetachedSignaturePolicyV1): + integrity_mutations = ( + replace(source.integrity, signature_url=source.integrity.signature_url + ".invalid"), + replace(source.integrity, signature_length=source.integrity.signature_length + 1), + replace(source.integrity, signature_sha256=sha256(source.integrity.signature_sha256)), + replace( + source.integrity, + public_key_packets_sha256=sha256( + source.integrity.public_key_packets_sha256 + ), + ), + replace( + source.integrity, + signer_fingerprint=bytes( + reversed(source.integrity.signer_fingerprint) + ), + ), + ) + else: + integrity_mutations = ( + replace(source.integrity, repository_url=source.integrity.repository_url + ".invalid"), + replace(source.integrity, tag=source.integrity.tag + "x"), + replace(source.integrity, commit=bytes(reversed(source.integrity.commit))), + replace(source.integrity, tree=bytes(reversed(source.integrity.tree))), + replace( + source.integrity, + omitted_paths=(source.integrity.omitted_paths[0] + "x",) + + source.integrity.omitted_paths[1:], + ), + replace( + source.integrity, + project_pinned_release_only_files=( + replace( + source.integrity.project_pinned_release_only_files[0], + sha256=sha256( + source.integrity.project_pinned_release_only_files[0].sha256 + ), + ), + ) + + source.integrity.project_pinned_release_only_files[1:], + ), + ) + for mutation in ( + *source_mutations, + *(replace(source, integrity=value) for value in integrity_mutations), + ): + sources = list(lock.sources) + sources[index] = mutation + changed = type(lock)(tuple(sources)) + self.assertNotEqual(changed.identity, lock.identity) + self.assertNotIn(changed.identity, seen) + seen.add(changed.identity) + + def test_parser_rejects_malleability_and_arbitrary_order(self) -> None: + lock = arb_source_lock_v1() + encoded = lock.encode() + for hostile in (encoded[:-1], encoded + b"\0", b"wrong!!!" + encoded[8:]): + with self.assertRaises(ProvenanceErrorV1): + type(lock).parse(hostile) + with self.assertRaises(ProvenanceErrorV1) as caught: + type(lock)((lock.sources[1], lock.sources[0], lock.sources[2])) + self.assertEqual(caught.exception.reason, ProvenanceReasonV1.NONCANONICAL_ORDER) + + with self.assertRaises(ProvenanceErrorV1) as caught: + type(lock).parse(encoded[:10]) + self.assertEqual(caught.exception.reason, ProvenanceReasonV1.TRUNCATED) + + def test_malformed_url_is_a_typed_input_failure(self) -> None: + for url in ( + "https://[invalid/signature", + "https://example.invalid:bad/signature", + "https://example.invalid/\nsignature", + ): + with self.subTest(url=url): + with self.assertRaises(ProvenanceErrorV1) as caught: + DetachedSignaturePolicyV1( + url, + 3, + sha256(b"sig"), + sha256(b"packets"), + bytes.fromhex("00112233445566778899aabbccddeeff00112233"), + ) + self.assertEqual( + caught.exception.reason, + ProvenanceReasonV1.INVALID_FIELD, + ) + + def test_constructor_cardinality_limits_match_the_wire_parser(self) -> None: + lock = fixture_lock(GOOD_ARCHIVE) + with self.assertRaises(ProvenanceErrorV1) as caught: + replace(lock, legal_files=lock.legal_files * 4_097) + self.assertEqual(caught.exception.reason, ProvenanceReasonV1.INVALID_FIELD) + + flint_integrity = arb_source_lock_v1().sources[2].integrity + self.assertIsInstance(flint_integrity, GitContentRelationPolicyV1) + with self.assertRaises(ProvenanceErrorV1) as caught: + replace( + flint_integrity, + omitted_paths=flint_integrity.omitted_paths * 205, + ) + self.assertEqual(caught.exception.reason, ProvenanceReasonV1.INVALID_FIELD) + + +class SafeArchiveAdmissionTests(unittest.TestCase): + def test_three_locked_sources_become_one_ordered_capability(self) -> None: + gmp = fixture_lock(GOOD_ARCHIVE) + mpfr = replace(gmp, role=SourceRoleV1.MPFR) + flint = replace( + gmp, + role=SourceRoleV1.FLINT_ARB, + integrity=GitContentRelationPolicyV1( + "https://example.invalid/fixture.git", + "v1", + bytes.fromhex("11" * 20), + bytes.fromhex("22" * 20), + 1, + ("missing",), + ( + ProjectPinnedReleaseOnlyFileV1( + "value", + 0o644, + 4, + sha256(b"data"), + ), + ), + ), + ) + lock = provenance.ArbSourceLockV1((gmp, mpfr, flint)) + sources = tuple( + admit_source_archive(expected, GOOD_ARCHIVE) + for expected in lock.sources + ) + + admitted = admit_arb_sources(lock, sources) + + self.assertIs(type(admitted), AdmittedArbSourcesV1) + self.assertEqual(admitted.source_lock_identity, lock.identity) + self.assertEqual(admitted.sources, sources) + self.assertEqual(len(admitted.identity), 32) + with self.assertRaises((ProvenanceErrorV1, TypeError)): + admit_arb_sources(lock, (sources[1], sources[0], sources[2])) + with self.assertRaises(TypeError): + AdmittedArbSourcesV1( + lock.identity, + sources, + _token=object(), + ) + + def test_archive_is_hash_checked_then_scanned_without_extracting(self) -> None: + lock = fixture_lock(GOOD_ARCHIVE) + admitted = admit_source_archive(lock, GOOD_ARCHIVE) + self.assertEqual(admitted.source_lock_identity, lock.identity) + self.assertEqual(admitted.regular_file_count, 2) + self.assertEqual(admitted.regular_file_bytes, 11) + self.assertEqual(tuple(item.path for item in admitted.files), ("LICENSE", "value")) + self.assertEqual(admitted.files[0].sha256, sha256(b"license")) + self.assertIs(admitted.archive_bytes, GOOD_ARCHIVE) + + with self.assertRaises(TypeError): + provenance.SafeSourceArchiveV1( + lock.identity, + lock.archive_sha256, + b"t" * 32, + 2, + 11, + admitted.files, + GOOD_ARCHIVE, + _token=object(), + ) + + for changed in ( + replace(lock, archive_length=lock.archive_length + 1), + replace(lock, archive_sha256=sha256(b"other")), + replace(lock, tar_stream_length=lock.tar_stream_length + 512), + replace(lock, root_prefix="other/"), + replace(lock, regular_file_count=3), + replace(lock, regular_file_bytes=12), + replace( + lock, + legal_files=(LegalFileV1("LICENSE", 7, sha256(b"wrong")),), + ), + ): + with self.assertRaises(ProvenanceErrorV1): + admit_source_archive(changed, GOOD_ARCHIVE) + + changed_legal_file = replace( + lock, + legal_files=(LegalFileV1("LICENSE", 7, sha256(b"wrong")),), + ) + with self.assertRaises(ProvenanceErrorV1) as caught: + admit_source_archive(changed_legal_file, GOOD_ARCHIVE) + self.assertEqual( + caught.exception.reason, + ProvenanceReasonV1.LEGAL_FILES_MISMATCH, + ) + + def test_unsafe_member_kinds_and_paths_are_rejected(self) -> None: + fixtures = ( + (ProvenanceReasonV1.UNSAFE_PATH, ( + ("fixture-1/", None, None), + ("fixture-1/LICENSE", b"license", None), + ("fixture-1/../escape", b"data", None), + )), + (ProvenanceReasonV1.UNSAFE_PATH, ( + ("fixture-1/", None, None), + ("fixture-1/LICENSE", b"license", None), + ("fixture-1/bad\\path", b"data", None), + )), + (ProvenanceReasonV1.ABSOLUTE_PATH, ( + ("fixture-1/", None, None), + ("fixture-1/LICENSE", b"license", None), + ("/absolute", b"data", None), + )), + (ProvenanceReasonV1.UNSAFE_LINK, ( + ("fixture-1/", None, None), + ("fixture-1/LICENSE", b"license", None), + ("fixture-1/link", None, b"../escape"), + )), + (ProvenanceReasonV1.CASE_COLLISION, ( + ("fixture-1/", None, None), + ("fixture-1/LICENSE", b"license", None), + ("fixture-1/license", b"data", None), + )), + (ProvenanceReasonV1.DUPLICATE_PATH, ( + ("fixture-1/", None, None), + ("fixture-1/LICENSE", b"license", None), + ("fixture-1/LICENSE", b"data", None), + )), + (ProvenanceReasonV1.UNSAFE_PATH, ( + ("fixture-1/", None, None), + ("fixture-1/a/b/", None, None), + ("fixture-1/LICENSE", b"license", None), + ("fixture-1/value", b"data", None), + )), + ) + for expected, entries in fixtures: + with self.subTest(expected=expected): + archive = tar_gz(entries) + lock = fixture_lock( + archive, + file_count=2, + unpacked_bytes=11, + ) + with self.assertRaises(ProvenanceErrorV1) as caught: + admit_source_archive(lock, archive) + self.assertEqual(caught.exception.reason, expected) + + def test_special_member_and_noncanonical_compressed_stream_are_rejected(self) -> None: + for member_type, reason in ( + (tarfile.FIFOTYPE, ProvenanceReasonV1.UNSAFE_MEMBER_TYPE), + (tarfile.CHRTYPE, ProvenanceReasonV1.UNSAFE_MEMBER_TYPE), + (tarfile.BLKTYPE, ProvenanceReasonV1.UNSAFE_MEMBER_TYPE), + (tarfile.LNKTYPE, ProvenanceReasonV1.UNSAFE_LINK), + ): + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode="w", format=tarfile.USTAR_FORMAT) as archive: + root = tarfile.TarInfo("fixture-1/") + root.type = tarfile.DIRTYPE + root.mode = 0o755 + archive.addfile(root) + license_member = tarfile.TarInfo("fixture-1/LICENSE") + license_member.size = 7 + license_member.mode = 0o644 + archive.addfile(license_member, io.BytesIO(b"license")) + hostile = tarfile.TarInfo("fixture-1/hostile") + hostile.type = member_type + hostile.mode = 0o644 + hostile.linkname = "fixture-1/LICENSE" + archive.addfile(hostile) + special = gzip.compress(raw.getvalue(), mtime=0) + with self.assertRaises(ProvenanceErrorV1) as caught: + admit_source_archive( + fixture_lock(special, file_count=1, unpacked_bytes=7), + special, + ) + self.assertEqual(caught.exception.reason, reason) + + concatenated = GOOD_ARCHIVE + gzip.compress(b"trailing", mtime=0) + with self.assertRaises(ProvenanceErrorV1) as caught: + admit_source_archive( + fixture_lock( + concatenated, + tar_stream_bytes=len(gzip.decompress(GOOD_ARCHIVE)), + ), + concatenated, + ) + self.assertEqual(caught.exception.reason, ProvenanceReasonV1.TRAILING_COMPRESSED_DATA) + + def test_xz_uses_the_same_bounded_archive_law(self) -> None: + raw_tar = gzip.decompress(GOOD_ARCHIVE) + archive = lzma.compress(raw_tar, format=lzma.FORMAT_XZ) + lock = fixture_lock(archive, archive_format=ArchiveFormatV1.TAR_XZ) + admitted = admit_source_archive(lock, archive) + self.assertEqual(admitted.regular_file_count, 2) + + def test_compressed_expansion_cannot_cross_the_locked_tar_bound(self) -> None: + raw_tar = gzip.decompress(GOOD_ARCHIVE) + locked_length = len(raw_tar) - 512 + for archive_format, archive in ( + ( + ArchiveFormatV1.TAR_GZIP, + gzip.compress(raw_tar, compresslevel=9, mtime=0), + ), + ( + ArchiveFormatV1.TAR_XZ, + lzma.compress(raw_tar, format=lzma.FORMAT_XZ), + ), + ): + with self.subTest(archive_format=archive_format): + lock = fixture_lock( + archive, + tar_stream_bytes=locked_length, + archive_format=archive_format, + ) + with self.assertRaises(ProvenanceErrorV1) as caught: + admit_source_archive(lock, archive) + self.assertEqual( + caught.exception.reason, + ProvenanceReasonV1.TAR_STREAM_LENGTH_MISMATCH, + ) + + def test_encoded_mutations_never_reenter_as_the_same_lock(self) -> None: + lock = arb_source_lock_v1() + encoded = lock.encode() + accepted = 0 + for offset in range(len(encoded)): + mutated = encoded[:offset] + bytes((encoded[offset] ^ 1,)) + encoded[offset + 1 :] + try: + parsed = type(lock).parse(mutated) + except ProvenanceErrorV1: + continue + accepted += 1 + self.assertNotEqual(parsed.identity, lock.identity) + self.assertEqual(parsed.encode(), mutated) + self.assertGreater(accepted, 0) + + +if __name__ == "__main__": + unittest.main()