diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4abf459..8dca340 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -213,3 +213,59 @@ jobs: - name: Run the unit tests # vitest run exits 1 when it finds no test file, so an empty run fails. run: npm run test:unit + + compatibility: + # The compatibility model in docs/CI-MATRIX.md: the validator's own tests, the committed + # profile, and a record of this runner checked against that profile. A pass shows that + # this tooling works on this runner; it is not a support claim for any other platform. + name: Compatibility / Schema & Probe / Linux x64 + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + # uv ignores uv.toml and pyproject.toml files, which a pull request could add. + UV_NO_CONFIG: '1' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + version: '0.12.24' + enable-cache: false + # With a Python version given, setup-uv does not look for an interpreter in the checkout and run it. + python-version: '3.12.15' + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + package-manager-cache: false + - name: Create a CPython 3.12.15 environment with no packages + # The scripts use the standard library only, so nothing is installed into it. + run: | + uv venv --managed-python --python 3.12.15 + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" + - name: Record this runner's environment + # Reads /proc, /sys, os-release, a few files under /run, the checkout's .git, the event file's + # head and base SHAs and allowlisted variables, and runs node -p; the collector's docstring + # lists every source. Never escalates privileges, and writes no record that looks like a secret. + run: | + mkdir -p "$RUNNER_TEMP/compatibility" + python -I scripts/ci/collect_environment.py --profile-id linux-x64-gha --output "$RUNNER_TEMP/compatibility/linux-x64-gha.result.json" --summary "$GITHUB_STEP_SUMMARY" + - name: Check the record and the committed profile + # Also after a probe failure, which writes a partial record and exits 1. + if: ${{ !cancelled() }} + run: python -I scripts/ci/validate_compatibility.py --require-real --dependency-strategy docs/DEPENDENCY-STRATEGY.md --profile tests/compatibility/profiles/linux-x64-gha.json --summary "$GITHUB_STEP_SUMMARY" "$RUNNER_TEMP/compatibility/linux-x64-gha.result.json" tests/compatibility/profiles/*.json + - name: Test the validator, including records that must fail + if: ${{ !cancelled() }} + # Run directly, not through unittest discover, so scripts/ci/tests/ is not on the import path. + run: python -I scripts/ci/tests/test_compatibility_schema.py --verbose + - name: Test the probe + if: ${{ !cancelled() }} + run: python -I scripts/ci/tests/test_environment_probe.py --verbose + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ !cancelled() }} + with: + name: compatibility-linux-x64-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/compatibility/ + if-no-files-found: warn + # The record is a few KB and is the lane's only machine-readable evidence: keep it 90 days, GitHub's default for logs. + retention-days: 90 diff --git a/docs/CI-MATRIX.md b/docs/CI-MATRIX.md new file mode 100644 index 0000000..bec40e5 --- /dev/null +++ b/docs/CI-MATRIX.md @@ -0,0 +1,262 @@ +# Claus CI Matrix and Compatibility Model + +Added 2026-10-11. This document defines how a compatibility lane is described, how a run on a lane is recorded, and what the validator checks. Commands and the CI job are listed in [TESTING.md](TESTING.md); which lanes are supported is recorded only in [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md#platform-and-accelerator-lanes); CI permissions and trust boundaries are in [SECURITY-ARCHITECTURE.md](SECURITY-ARCHITECTURE.md) "CI workflow". + +Implemented today: the schema, the validator, the environment collector, one real profile, nine synthetic fixtures, their tests, and one CI job on the checked lane (GitHub-hosted `ubuntu-24.04` x86-64). No other lane runs in CI. Nothing in this model is a support claim, and a synthetic fixture is not evidence that any platform works. + +## Files + +| Path | Role | +|---|---| +| `tests/compatibility/schema.json` | Structure of profiles and results, written in the JSON Schema subset described under "Structure" below. | +| `tests/compatibility/profiles/` | Real profiles: one per lane that a CI job records. Today only `linux-x64-gha.json`. | +| `tests/compatibility/fixtures/` | Synthetic profiles and results for scenarios no job runs (Windows on Arm, WSL2, Podman, Kubernetes, CUDA). They test the model, not a platform. | +| `scripts/ci/collect_environment.py` | Records the environment of the process that runs it as a result document. | +| `scripts/ci/validate_compatibility.py` | Checks profiles and results: structure from the schema, then the rules under "Rules across fields". | +| `scripts/ci/tests/test_compatibility_schema.py`, `scripts/ci/tests/test_environment_probe.py` | Tests of the validator and the collector, including documents that must fail. | + +All of these use the Python standard library only and run without Django. + +## Profiles and results + +A **profile** declares a lane: the layers it runs on, the runtimes it uses, its compute backends (and, for an accelerator lane, the backend its workload stages must run on), the stages a run must pass, its privilege policy, the CI job that records it, and its support status. A profile holds only `declared`, `unknown`, and `not-applicable` facts; it cannot claim to have observed anything. + +A **result** records one run: what the collector observed on the machine, the stages and their status, and the CI run and commits it belongs to. `schema_version` is 1 for both; `kind` is `profile` or `result`. + +A document with `synthetic: true` was never measured. All its evidence references start with `synthetic:`, a synthetic profile's `id` starts with `synthetic-`, and a synthetic result has `generated_at: null` and names no run, commit, or CI system (every `run` fact is `unknown` or `not-applicable`). `--require-real` rejects synthetic documents, and a real result can only be checked against a real profile. + +## Axes + +Each axis is a separate field. Host, guest, container, and workload are never merged into one OS or CPU value. + +| Axis | Where it is recorded | +|---|---| +| Execution layers | `environment.layers`, outermost first, 1 to 8 layers. `kind` is `host`, `vm`, `wsl2`, or `container`; `technology` names the hypervisor or engine (`kvm`, `hyper-v`, `wsl2`, `podman`, `containerd`, ...). | +| OS per layer | `os.family`, `os.name`, `os.version`, `os.kernel`. A container layer has no kernel of its own. | +| CPU per layer | `cpu.arch` and `cpu.execution` (`native` or `emulated`). | +| Service manager per layer | `service_manager.name` and `scope`, for example systemd inside a container, or a host systemd running Quadlet units with user scope. | +| Container engine and orchestration | `container.engine_mode` (`rootless` or `rootful`), `privileged`, `devices`, `orchestration.name`, `version`, and `scope`, on container layers only. | +| Workload identity | `workload.identity`: `uid` (the effective user ID), `real_uid`, `saved_uid`, `gid`, `superuser`, and the user namespace (`initial`, `uid_map`). | +| Workload isolation | `workload.isolation`: `no_new_privs`, effective, permitted, and bounding capabilities, seccomp mode, AppArmor, SELinux, cgroup version and delegation, and whether the process is in the initial PID namespace. | +| Runtimes and ABI | `workload.runtimes.python` (implementation, version, cache tag, SOABI, platform tag, build architecture, pointer size, free-threaded build, GIL state, libc, execution) and `workload.runtimes.node` (version, module ABI, N-API version, platform, architecture, execution). In a collected record, `gil_enabled` is the state of the collector's own process, which imports only the standard library; it does not show the GIL state of an application or of a later stage. | +| Compute backends | `compute.backends`: each has an `id`, a `kind` (`cpu`, `gpu`, `npu`, `other`), an `api`, availability, device, version, and features. A CPU-only lane lists one `cpu` backend. The `cpu` API is always of kind `cpu`; a backend of kind `cpu` may use another API, such as `wasm`. | +| Run and revision | `run.ci` (system, event, repository, ref, run ID and attempt, run URL, workflow ref, job, runner environment, OS, architecture, image) and `run.revision` (see "Commits" below). | +| Stages | `stages`: the seven stages below, each with its status. | + +Documented values: architectures `x86_64`, `aarch64`, `x86`, `armv7`, `riscv64`, `ppc64le`, `s390x`, `loongarch64` (aliases such as `amd64` or `arm64` are errors; write the canonical name); OS families `linux`, `windows`, `macos`, `freebsd`; backend APIs `cpu`, `cuda`, `rocm`, `oneapi`, `level-zero`, `opencl`, `vulkan-compute`, `directcompute`, `directml`, `metal`, `mps`, `coreml`, `qnn`, `wasm`, `webgpu`, `webgl`, `webnn`; and the technologies, orchestrators, and service managers listed in the validator. Any other value is accepted with an `unrecognized-value` warning. Rules that need a documented value (which architectures run natively on which, pointer sizes, the targets of platform and ABI tags) do not apply to it, except that an ABI tag for an architecture outside the list disagrees with a listed arch fact; rules that only compare two values for equality, such as a container's OS family with its parent's, still do. An unrecognized value is never treated as supported. Node's module ABI and N-API version are recorded but not compared with its version. + +`extensions` objects with `x-` keys may appear on documents, layers, backends, and stages for data the schema has no field for. The validator checks their strings for control characters and secret-shaped values and nothing else: data inside them that looks like a fact or an evidence list is not read as one. There are no per-technology validators. + +### Layers and nesting + +Layers run from the outermost (index 0) inwards; the workload runs in the last one. The validator requires: + +- A `host` is only ever index 0, and a `vm`, `wsl2`, or `container` always has a layer around it. When the outer layer cannot be seen, it is still listed, with `unknown` facts and a reason. +- A container shares the kernel of the layer below it, so it has the same OS family and records no kernel. +- WSL2 runs on a Windows layer and is itself Linux. +- Only container layers carry container facts, and every container layer carries them. +- Each documented technology belongs to one kind of layer: the hypervisors (`kvm`, `hyper-v`, `qemu`, ...) to `vm`, the engines (`docker`, `podman`, `containerd`, ...) to `container`, and `wsl2` to `wsl2`. A documented technology on a layer of another kind is an error, and a `wsl2` layer names no technology other than `wsl2`. The outermost `host` runs on hardware and names none; a `vm`, `wsl2`, or `container` layer is made by something, so its technology is never `not-applicable` (`unknown` when it cannot be seen). + +Examples, from the profile and the fixtures: + +| Scenario | Layers | +|---|---| +| GitHub-hosted runner (`linux-x64-gha` profile) | host (all facts unknown) / vm (Ubuntu 24.04, x86-64, systemd; the hypervisor is left unknown) | +| Windows on Arm, native | host (Windows 10.0.26100, aarch64) | +| Windows on Arm, WSL2, rootless Podman | host (Windows) / wsl2 (Linux, aarch64) / container (Ubuntu 24.04, Podman, rootless) | +| VM with a Podman container running systemd | host / vm (KVM, Fedora 42, systemd) / container (Podman, rootless, systemd inside; the VM's systemd runs it as a user-scope Quadlet unit) | +| Kubernetes pod with CUDA on aarch64 | host (Ubuntu 24.04, aarch64) / container (containerd, rootful, orchestration kubernetes 1.33, device `nvidia-gpu`) | + +### Facts and provenance + +Every measurable value is a fact: `{value, provenance, evidence, reason}`. + +| Provenance | Meaning | Required | +|---|---|---| +| `observed` | Read on the machine by the collector. | A value and at least one evidence reference. | +| `inferred` | Derived from named evidence. | A value and evidence. | +| `declared` | Stated by a profile, a workflow, or a tool, not measured. | A value. | +| `unknown` | Not established. | `value: null` and a reason. | +| `not-applicable` | Has no meaning here. | `value: null` and a reason. | + +Only `observed` and `inferred` facts are measurements. A `declared` value in a result is a statement: the validator checks it for contradictions, but it cannot satisfy a policy rule or a fact a profile declares. The exception is a stage's `privilege.escalation`, which the job that ran the stage states, because the collector cannot see the privileges of another step. + +Evidence references have the form `:`, with kind `file`, `env`, `python`, `cmd`, `event`, `git`, `tool`, `doc`, `profile`, `workflow`, `log`, `artifact`, or `synthetic`. A reference names where a value came from; it never holds the value of a secret or the contents of a file. + +### CPU execution + +`native` means no translation of the instructions was found, not that the hardware was shown to run them; `emulated` means some layer translates the instructions. The collector compares the architecture a process was built for with the kernel's own architecture from `/proc/sys/kernel/arch` (Linux 6.1 and later), which qemu-user's `uname` interception does not rewrite. The Python architecture comes from the interpreter's build triplet (`sys.implementation._multiarch`, then `SOABI`, then the ELF header of the executable, read only when its byte order and ELF class are those of the architecture its `e_machine` names), not from `sysconfig.get_platform()`, which on Linux follows `uname`. Without `/proc/sys/kernel/arch`, a process is reported `emulated` only when `/proc/cpuinfo` shows another CPU family, and `unknown` otherwise. `x86` code on an `x86_64` kernel is native. `armv7` code on an `aarch64` kernel is native only on cores that implement AArch32, and many recent Arm cores implement AArch64 only, so the collector records that pair as `unknown`; the validator accepts either value for it. On a kernel without `/proc/sys/kernel/arch`, such as any kernel before 6.1 that has not backported it, the layer's `cpu.arch` and most runtimes' `execution` stay `unknown`, so a profile for such a lane leaves them `unknown` too. + +Emulation is inherited. The validator rejects a layer or a runtime recorded as `native` when an enclosing layer is recorded as `emulated`, or when its architecture cannot run natively on an enclosing layer whose architecture is known, not only the one directly below it. + +Limits: a whole system emulated by its hypervisor looks native from inside; translators that also present the emulated architecture as the kernel's (such as FEX or box64 setups, or Rosetta inside a Linux VM) have not been evaluated; and the layer that runs the kernel records its own `cpu.execution` as `unknown`, because a guest cannot see whether the layer below emulates it. A container layer's `cpu.arch` and `cpu.execution` are `unknown` in a collected record: the container runs on the CPU of the layer that runs the kernel, and the programs in it need not all be built for one architecture, so each runtime records its own architecture and execution. A profile or a synthetic record may still give a container layer `cpu` facts; the rules above apply to them as to any other layer's. + +## Privilege model (Rootless First) + +These facts are independent, and none is derived from another: + +| Fact | Field | Note | +|---|---|---| +| The workload's user | `workload.identity.superuser`, `uid`, `real_uid`, `saved_uid` | On Linux, macOS, and FreeBSD the superuser is effective user ID 0, inside its own user namespace on Linux. A real or saved user ID of 0 lets the process set its effective ID back to 0, so `app-non-root` reads all three. On Windows the superuser is a process with an elevated token, in the Administrators group, or running as SYSTEM, and `real_uid` and `saved_uid` are `not-applicable`. | +| User namespace | `user_namespace.initial`, `uid_map` | `initial` is read from the inode of `/proc/self/ns/user`. An identity `uid_map` (`0 0 4294967295`) does not prove the initial namespace, because a nested namespace can have one too. | +| PID namespace | `isolation.pid_namespace.initial` | Read from the inode of `/proc/self/ns/pid`. Most containers, and every WSL2 distribution, get their own PID namespace, so a process outside the initial one may be in a container. The initial one does not rule a container out: Apptainer by default, a container given the host's PID namespace, and a chroot share it, and are seen only through the markers the collector reads. | +| Engine mode | `container.engine_mode` | Whether the engine runs without root on its host. An image can ship its own `/run/.containerenv`, so the collector reads the mode from that file only as `inferred`, and only when every engine marker it reads names Podman. | +| Privileged container | `container.privileged` | | +| Capabilities | `isolation.capabilities.effective`, `permitted`, `bounding` | A permitted capability can be made effective again, so the policy reads both sets. | +| No new privileges, seccomp, AppArmor, SELinux | `isolation.*` | AppArmor's `enabled` parameter counts only as `Y` or `N` and a newline, as the kernel writes it (any other value, also a padded or longer one, is `unknown`), and its label is read from `/proc/self/attr/apparmor/current` only; SELinux's context only when SELinux is the first major security module listed. | +| cgroup version and delegation | `isolation.cgroup` | The version is the file system mounted at `/sys/fs/cgroup`; delegation is read for cgroup v2 from write permission on the process's own cgroup directory, its `cgroup.procs`, its `cgroup.threads` where it exists (Linux 4.14 and later), and its `cgroup.subtree_control` (without the last, no controller can be enabled below it), and only when none of its real, effective, or saved user IDs is 0 and it holds no effective or permitted capability, because `access(2)` checks the real user ID. | +| Devices | `container.devices` | Devices the engine passes in. | +| Provisioning and runtime privilege | `stages..privilege.escalation` | `none`, `sudo`, `root-login`, `setuid`, or `other`, per stage. An `install` that used `sudo` and an `execute` that used `none` are recorded as such. | + +A rootless engine does not make the process inside the container non-root: uid 0 in a rootless Podman container is still the superuser of its namespace, and `app-non-root` fails for it. A root process inside a container does not make the engine rootful. + +### Policy + +| Rule | Default | Checks | +|---|---|---| +| `app-non-root` | required | `superuser` is false, and neither the real nor the saved user ID is 0. On a measured OS family other than Linux, macOS, or FreeBSD, `not-applicable` user IDs pass. | +| `engine-rootless` | preferred | Every container layer's engine is rootless. | +| `no-privileged-container` | required | Every container layer is unprivileged. | +| `no-privilege-escalation` | required | Every stage that ran used no escalation. | +| `minimal-capabilities` | required | The workload holds no effective and no permitted capabilities. | +| `minimal-devices` | required | No device is passed to a container. | + +A rule passes only when the record shows compliance with `observed` or `inferred` facts: an `unknown` or `declared` value counts as a violation, except a stage's escalation, which the job states (see "Facts and provenance"). `minimal-capabilities` checks the effective and permitted sets the workload holds. The bounding set, which limits what a setuid program could gain, is recorded but not checked, because a non-root process on an ordinary host has the full bounding set. Inside a user namespace other than the initial one, capabilities apply to that namespace only; the rule still counts them. A violation of a required rule is an error, and of a preferred rule a warning. A profile can raise a rule from preferred to required; it cannot lower a required rule (`policy-weakened`). `not-applicable` effective or permitted capabilities pass `minimal-capabilities` only when the innermost layer's OS family is measured and is not Linux. Windows has no Linux capabilities, but the model does not record Windows token privileges either, so a Windows record with `not-applicable` capabilities passes this rule without showing that its token holds no privilege. Off Linux the collector records the isolation facts as `unknown` and, on Windows, `superuser` as `unknown`, so a collected macOS or Windows record fails `minimal-capabilities`, and on Windows `app-non-root`, until a probe for that system records them. + +A layer that any fact, even a declared one, calls a container is held to the container rules, which only measured facts pass. The container rules also apply when no layer is a container but the record does not show the workload outside one. On Linux, or when the innermost layer's OS family is not measured, unless `pid_namespace.initial` is measured as true, `no-privileged-container` and `minimal-devices` fail and `engine-rootless` warns at `/workload/isolation/pid_namespace/initial`, because a container that leaves none of the markers the collector reads would otherwise pass. No exception covers this finding; record the container layer instead, so that an exception can cover that layer. The rule does not apply when the innermost layer is `wsl2`, because WSL2 runs every distribution in its own PID namespace inside its VM, so there the namespace does not suggest a container; a container in a WSL2 distribution that leaves no marker is therefore not caught. Nor is a container that shares the initial PID namespace and leaves no marker. On another measured OS family no namespace probe applies, so the innermost layer's `kind` must be measured as `host` or `vm`; otherwise the same three findings are reported at that `kind`. The collector records the kind as `unknown` off Linux, so a collected macOS or Windows record fails these rules too. + +Inside a container the collector cannot see the privileged state or the devices, and Podman writes the engine's rootless state to `/run/.containerenv` only for privileged containers, so a container lane's record fails the default policy until its job records those facts from the engine side, outside the container; no job does that yet. Docker writes no such file, and an image can contain one, so the collector names no engine and no mode when its markers disagree. + +A profile that needs to break a rule records an exception with `id`, `rule`, `reason`, `scope`, `verification`, `end_condition`, and `approval` (`approved`, `pending`, or `rejected`, with a reference and a date when approved; a date given must be a day that exists). The scope names every value the exception covers, under the keys its rule uses and no other: `layers` for `engine-rootless` and `no-privileged-container`, both `stages` and `escalations` (`sudo`, `root-login`, `setuid`, `other`) for `no-privilege-escalation`, `capabilities` for `minimal-capabilities`, and `devices` for `minimal-devices`; `app-non-root` takes an empty scope. A violation is covered only when each of its values is listed, so an exception for `sudo` in `install` does not cover `setuid` there, and no exception covers a state that is not measured. Only an approved exception covers a violation; a pending or rejected one leaves the error (`policy-exception-not-approved`), and an approved one that covers nothing gives a warning. The validator checks that the approval is recorded, not that it is genuine, so every use of an approved exception gives a `policy-exception-applied` warning that names it, what it covers, and its approval reference and date; reviewing that reference is a reviewer's step ([SECURITY-REVIEW.md](SECURITY-REVIEW.md)). The approval has no approver or expiry field: the reference and the `end_condition` carry them. A profile cannot hold observed facts, so an exception cannot be used to state facts a run did not show. `--no-policy` skips these rules and reports a `policy-skipped` warning. + +## Stages and statuses + +| Stage | What it covers | Needs | +|---|---|---| +| `detect` | The environment was recorded. The collector sets it. | | +| `install` | The lane's tools and dependencies were installed. | `detect` | +| `initialize` | The application or runtime started and loaded its configuration. | `install` | +| `execute` | The workload ran. | `initialize` | +| `correctness` | Its results were checked against expectations. | `execute` | +| `security` | Security checks ran on this lane. | `detect` | +| `performance` | It was measured against a budget. | `correctness` | + +Statuses: `passed`, `failed`, `blocked` (could not run because a prerequisite failed or something it needs is missing), `not-run` (nobody ran it), and `not-applicable`. Only `security` and `performance` can be `not-applicable`: every workload is detected, installed, initialized, run, and checked, so those stages are `not-run` or `blocked` when nobody ran them. A `passed` or `failed` stage cites the evidence that shows it and records its privilege escalation; in a real result that evidence includes a `log:` or `artifact:` reference, and `detect` may cite the collector (`tool:`) instead. Every status other than `passed` gives a reason; a `not-run` stage cites nothing; a stage can be `passed` or `failed` only when every stage it needs is `passed` or `not-applicable`; and `detect` cannot be `passed` while `probe_errors` lists a failed probe. + +`overall` is derived: `failed` if any stage failed, `passed` if every stage is `passed` or `not-applicable` and at least one passed, and `incomplete` otherwise. So a `passed` run has passed every stage from `detect` to `correctness`. The current CI job runs `detect` only, so its records are `incomplete` by design. + +A stage that used a compute backend records `compute.requested` and `compute.effective` as backend IDs. A stage cannot pass when a `gpu` or `npu` backend was requested and another backend, or none, ran (`accelerator-fallback`): record such a run as `failed` with the reason, or request the CPU backend for a CPU run. A profile of an accelerator lane names its `gpu` or `npu` backend in `accelerator_backend`; then each of `execute`, `correctness`, and `performance` that the profile requires must record that backend as both requested and effective (`required-stage`), so a run that used only the CPU, or recorded no backend, does not satisfy the lane. + +### Stage status and support status + +Stage status describes one run. Support status describes a lane and is recorded only in the [lane table](DEPENDENCY-STRATEGY.md#platform-and-accelerator-lanes), whose words map as follows: Checked to `checked`, Statically resolvable to `statically-resolvable`, Not supported, not binary-installable, or incomplete to `not-supported`. A real profile copies that status into `support.status` with the lane's name, written exactly as the table's row. A status the table gives for some versions only, written in its cell as ` or newer` or `or older`, also needs those versions declared and met: Linux x86-64 glibc is `checked` for glibc 2.39 or newer and CPython 3.12.13 or newer, and macOS Apple Silicon is `statically-resolvable` for macOS 15 or newer and `not-supported` for macOS 14 or older. The macOS version is the innermost layer's OS version, compared on its major number; the glibc version is read from the Python's `libc` fact written as the collector writes it (`glibc 2.39`, in any ASCII letter case), not from the SOABI; and the CPython version is a release version such as `3.12.15`, so `3.12` alone, a pre-release, or a build after a release such as `3.12.14+` meets no CPython bound. The validator keeps these bounds itself, and a test keeps them equal to the cells. With `--dependency-strategy`, the validator fails a profile whose lane is not in the table or whose status the table does not give, and a synthetic profile cannot claim `checked`; a table that lists a lane twice, whose status would then depend on the row order, is a usage error. + +A profile is bound to the lane it names. The validator knows what each row describes: the innermost layer's OS family, the architecture and C library the Python is built for (the C library from the `libc` fact, or from the SOABI when that fact has no value; a fact that starts with `glibc` or `musl` in any ASCII letter case, with no letter after it, names that library, and any other fact or a SOABI such as `-linux-android` names another C library, which contradicts a glibc or musl lane), and whether a WSL2 layer is present. A declared value that contradicts the lane is a `support-claim` error whatever the status. A status other than `not-assessed` also needs each of those values declared, and the interpreter of the backend lock, CPython 3.12 with the GIL, because the table's statuses are for that lock (the version is read whole, so `3.12-not-python` names no interpreter); a profile for another interpreter, such as 3.15 or 3.15t, names the lane and records `not-assessed`. `not-assessed` is also for a lane the table does not list, with no name, for a synthetic profile, which assesses nothing and may still name the lane it models, and for a profile whose versions meet no status of a row that has bounds, or that does not declare them; a real profile whose platform and interpreter are what its lane covers, and whose versions meet the bounds of one of the row's statuses where it has any, copies that status instead. The binding does not cover the distribution or its version: a row's evidence rests on particular systems (the Linux x86-64 glibc row on glibc 2.39 and 2.41), and a profile of the same platform and versions on another distribution is not told apart from them. A row without a glibc bound, such as a statically resolvable Linux row, takes any glibc version. A test keeps the validator's list of lanes equal to the table's rows, so a new row needs its description in the validator. + +No run changes support status. A passed stage, a green job, or a new profile never promotes a lane; promoting one, for example to Checked, is a reviewed change to [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md) with its evidence. `linux-x64-gha` copies its status from the table's Linux x86-64 glibc row, which rests on earlier installs, not on the compatibility job. + +## Validation + +### Structure: a JSON Schema subset + +The validator interprets these keywords and no others: `type`, `enum`, `const`, `properties`, `required`, `additionalProperties`, `propertyNames`, `items`, `minItems`, `maxItems`, `minLength`, `maxLength`, `pattern`, `minimum`, `maximum`, `$ref`, `if`, `then`, and `else`, with `$schema`, `$id`, `$comment`, `$defs`, `title`, and `description` accepted as annotations. `$schema` must name the 2020-12 dialect, and `$schema`, `$id`, and `$defs` may appear only at the root. A schema that uses any other keyword (such as `oneOf`, `anyOf`, `allOf`, `not`, `format`, `patternProperties`, `uniqueItems`, or `unevaluatedProperties`) is rejected before any document is read, with exit status 2, so the schema cannot ask for a check the validator does not perform. It is not a general JSON Schema implementation. Where its result can differ from a JSON Schema 2020-12 validator: + +- `$ref` resolves only `#/$defs/`, and `$defs` is allowed only at the root. +- `pattern` is matched with Python's `re.search`, not the ECMA-262 dialect. A pattern that uses one of these forms, which the two read differently, is rejected with the schema: an escaped letter other than `\t`, `\n`, `\r`, `\f`, `\v`, `\x`, or `\u` (such as the class escapes `\d`, `\w`, `\s`, and `\b`, the anchors `\A` and `\Z`, or Python's `\a`, `\N`, and `\U`), a `\u` escape of a surrogate (`\uD800` to `\uDFFF`, a code point in Python and a UTF-16 code unit in ECMA-262), a group that starts with `(?` other than `(?:`, a quantifier `{,n}`, a possessive quantifier such as `*+`, or a character class that opens with `]` or `^]`. Some of these forms are also matched inside a character class, where they are portable, so such a class is rejected too. The check looks for these forms only, so a new pattern still needs review. Python's `$` also matches before a final newline; such a value is still rejected by `control-character`. +- A number written with a fraction or an exponent, such as `1.0`, is a `number`, not an `integer`, also where a `type` decides an `if`. +- The rules across fields read the shape the default schema requires. A schema given with `--schema` that accepts a document those rules cannot read gives exit status 2, not a verdict. With the default schema, however its path is written, such an error is raised as the bug it is. +- A `$ref` that leads back to itself without moving to another value is rejected with the schema. +- After a `type` mismatch, the other keywords of that subschema are not applied, so one finding is reported for it. +- Before the schema runs, a document is rejected (`json`) if it is larger than 2 MiB (no more than that and one byte is read), is not UTF-8, starts with a byte order mark, repeats an object key, holds `NaN`, `Infinity`, or a number outside the double range, or nests too deeply. + +The rules across fields run only on a document whose structure is valid; otherwise a `semantic-skipped` warning says so. + +### Rules across fields + +Each finding has a stable code. Errors make the document invalid; warnings do not. + +| Code | Level | Catches | +|---|---|---| +| `schema` | error | A structure error from the schema. | +| `json` | error | A file that cannot be read as strict JSON. | +| `control-character`, `secret-like-value` | error | A control or bidirectional formatting character or a lone surrogate, or a value shaped like a GitHub, AWS, Slack, or JWT token or a private key. | +| `fact-provenance` | error | A fact whose value, evidence, or reason does not fit its provenance. | +| `profile-provenance` | error | An observed or inferred fact in a profile. | +| `synthetic-marking` | error | Synthetic and real data mixed, a synthetic result that names a run, commit, collection time, or CI system, a real result without a collection time, a `generated_at` that is not a valid date and time (such as `2026-02-30T12:00:00Z`; seconds run from 00 to 59, so a leap second is refused too), or a synthetic document where `--require-real` asks for a measured one. | +| `arch-name` | error | An alias instead of the canonical architecture name. | +| `unrecognized-value` | warning | A value outside the documented lists, or a CPython SOABI whose form the validator cannot read, so its OS and architecture are not checked. | +| `layer-structure` | error | Layer order, container facts, or a layer's technology that break the nesting rules. | +| `os-contradiction` | error | A container with another OS family than its parent, WSL2 off Windows or not Linux, or a runtime or CPython ABI tag built for another OS than the innermost layer, or, when that layer's OS family is unknown, a CPython platform tag and SOABI built for different OSes. | +| `cpu-contradiction` | error | Emulated execution declared native: a layer, Python, or Node recorded as native inside an emulated layer, or whose architecture cannot run natively on an enclosing layer; or an emulated outermost host. | +| `abi-contradiction` | error | A Python platform tag, architecture, pointer size, cache tag, SOABI (its version, `t` suffix, architecture, and C library, which must match the libc fact when either names glibc or musl; the libc fact is read as for support claims, so `Ubuntu GLIBC 2.39` names another C library), free-threading flag, or GIL state that disagree, or a glibc or musl libc fact for a Python whose innermost layer or platform tag is Windows, macOS or FreeBSD, or whose SOABI is Windows or macOS (a FreeBSD SOABI such as `cpython-312` names no system). When the architecture fact is a listed name, an ABI tag for an architecture outside the list, such as `vax`, disagrees with it too. A CPython version is written as `platform.python_version()` gives it (`3.12.15`, `3.13.0rc1`, or `3.14.0a1+` for a build after a release) or as major.minor; in either form no number has a leading zero, and any other text, such as `3.12-not-python`, is an error, because the version would otherwise be read from its first two numbers. On Linux the platform tag follows the kernel, so a 32-bit build on a 64-bit kernel of its family agrees with it; on Windows and macOS the tag names the build. When the architecture fact is unknown or not a listed name, the platform tag and the SOABI's architecture, if both name one OS, are compared with each other by the same rule. The cache tag, and the SOABI's version, suffix, OS, architecture and C library, are compared with the other facts for CPython only; whatever the implementation, the SOABI's OS still counts for the glibc and musl rule, and its C library for support claims. | +| `identity-contradiction` | error | `superuser` that disagrees with uid 0 on Linux, macOS, or FreeBSD, or an initial user namespace without the identity map. | +| `backend-reference` | error | Duplicate backend IDs, a backend with the `cpu` API whose kind is not `cpu`, a stage naming a backend that is not listed, a backend used by a stage that did not run, a passing stage on a backend not recorded as available, a passing stage whose compute record names no backend that ran (on a GPU or NPU request that is an `accelerator-fallback`), or a profile's `accelerator_backend` that is not one of its `gpu` or `npu` backends. | +| `accelerator-fallback` | error | A GPU or NPU stage that passed on another backend. | +| `stage-fields`, `stage-order` | error | A stage without its evidence, reason, or privilege record, a real stage that cites no log or artifact, a not-run stage with evidence, a stage other than `security` or `performance` recorded as not-applicable, `detect` passed with probe errors, or a stage that ran before its prerequisites passed. | +| `overall-mismatch` | error | `overall` that the stages do not give. | +| `ci-evidence` | error | A GitHub Actions record without its run (its event, repository, run ID and attempt, run URL, workflow ref, and job), the commit its checkout holds, its `GITHUB_SHA`, or, for a pull request, its head and base, each observed or inferred (a declared value does not count), or with a run URL for another repository, run or attempt (a URL with no `/attempts/` names the run, whatever its attempt, and an attempt of more than 10 digits names none), or a run attempt of 0. | +| `policy-` | error or warning | A policy violation with no approved exception; a warning for a preferred rule. | +| `policy-exception`, `policy-exception-not-approved`, `policy-weakened` | error | A malformed or unscoped exception, a violation covered only by an unapproved exception, or a profile that lowers a required rule. | +| `policy-exception-applied` | warning | An approved exception that covered a violation, with what it covered and its approval reference and date, for a reviewer to check. | +| `policy-exception-unused` | warning | An approved exception that covers nothing in the result. | +| `policy-skipped` | warning | `--no-policy` was given, so the verdict says nothing about the policy. | +| `support-claim` | error | A support status that the lane table does not give for the lane, a lane the table does not list or the validator does not know, a claim that names no lane, `checked` in a synthetic profile, a profile whose OS family, architecture, C library, or WSL2 layer contradicts its lane, a status other than `not-assessed` without those values or for an interpreter other than the lock's, a status whose version bounds the profile's declared versions do not meet or that it does not declare, or `not-assessed` in a real profile whose platform, interpreter, and versions its lane covers. | +| `support-claim-unverified` | warning | A `checked` claim validated without `--dependency-strategy`. | +| `evidence-source` | error | A real profile's evidence source that names a missing workflow file, one the validator cannot read (a file that is not UTF-8, or a path the system refuses, such as a name over its length limit), or a job its `jobs:` mapping does not define, or a real profile with no evidence source. | +| `profile-link` | error | A result checked against a profile it does not name, a real result against a synthetic profile or the reverse, a real profile checked with a record not measured as a GitHub Actions run, or a GitHub Actions record whose workflow and job are not one of the profile's evidence sources: the workflow is the path `run.ci.workflow_ref` names inside `run.ci.repository`, so a record that does not name both, or whose workflow ref is outside that repository or names no workflow file or no ref, links to no profile. | +| `profile-mismatch`, `profile-unverified` | error | A layer count that differs from the profile, a backend that is missing or has another kind or API, any other fact the profile declares (in a layer, a runtime, a backend matched by ID, the accelerator inventory, or the runner) that the result contradicts, a fact the profile declares `not-applicable` that the result measured, or a declared fact the run did not measure. | +| `required-stage` | error | A stage the profile requires that did not pass, or, under a profile with an `accelerator_backend`, a required `execute`, `correctness`, or `performance` stage that did not run on it. | +| `profile-unchecked` | warning | A result naming a profile that was not given; the default policy applies unless `--no-policy` is given. | + +The validator prints each finding on one line, with control characters replaced and `##[`, which the GitHub Actions runner reads as a workflow command anywhere in a line, written as `## [`, and `::`, which starts a workflow command at the start of a line, written as `: :`; a line it prints starts with a file name or with its own text, never with a value from a record, and a file name that starts with `::` cannot start a command either. A value shaped like a secret, and the rest of its line, is replaced by a note naming the shape, so a finding that echoes a rejected value, or a file name, never shows it. No shape needs a word boundary before it, so an escape the echo writes just before a value (`\n` from `repr`, `~1` in a JSON Pointer) does not hide one; the collector's check of its JSON text reads the shapes the same way. A finding names its place with a JSON Pointer. The `schema`, `control-character`, `secret-like-value`, `fact-provenance`, and `profile-provenance` rules, and `synthetic-marking` on an evidence reference, build the path as they walk the document, and that path is bounded: a key longer than 128 characters once escaped is written as its length before escaping and the first 8 hexadecimal digits of the SHA-256 of its UTF-8 bytes (a lone surrogate encoded as it is), such as `~(300 characters, sha256 1a2b3c4d)`, and a path of more than 16 keys keeps its first 8 and last 7 around `~...`. The other rules, such as `synthetic-marking` on a synthetic run, `profile-mismatch`, and `profile-unverified`, write the path of the place they check from the keys they read, without that bound. Outside `extensions`, the default schema allows only the property names it defines, which are short and hold no `/` or `~`, and no place deeper than 8 keys, and the other rules name no place inside `extensions`. Under the default schema, a long key or deep nesting therefore leaves the path of each finding at most 16 keys of at most 128 characters each, before the line's own escapes, though a document with many findings still prints a line for each. Under any other schema given with `--schema`, a path the other rules write can repeat a key as it is, unescaped and unshortened, and can hold more than 16 keys. An escaped key holds `~` only before `0` or `1`, so neither marker reads as a key, but a shortened path can name more than one place, also for two long keys of one length whose digests start alike: it names the place for a reader and is not resolved. Its `--summary` handles every table cell the same way and escapes Markdown. The collector writes its own lines, which name the output file, the same way. In both scripts, a command-line error, which repeats the rejected argument and the program name, is written the same way too. + +### Detection failures and validation failures + +These are kept apart. A file the collector cannot read gives an `unknown` fact with a reason, not an error. An unexpected exception inside one probe is recorded in `probe_errors` with the probe's name; the other probes' values stay in the record, `detect` becomes `failed`, and the collector exits 1 after writing the record. The validator then reports the failed `detect` as `required-stage` against a profile that requires it; the CI job runs the validator after a probe failure too. A record that breaks a rule fails with that rule's code. The collector exits 2 on a usage error, when it cannot write the record or the summary, when the record would hold a value shaped like a secret, which it then does not write, or when `--summary` names the record's own file, however that path is written, which it then leaves as written without appending the summary. The validator exits 2 on a usage error or an unusable file, schema, lane table, or summary. That includes a lane table that lists a lane twice, and a `--summary` that names the schema, the profile, the lane table, or a document, however that path is written, when it writes nothing, or a workflow file that a real profile of the run names as its evidence source, also one that is missing, which it then does not create; it can tell this only after reading the profiles, so it prints each document's findings but appends no summary. + +## Collector + +The collector reads `/proc`, `/sys`, `/etc/os-release` (or `/usr/lib/os-release`), `/run/.containerenv`, and `/run/systemd/container`; checks whether `/.dockerenv`, `/.singularity.d`, and `/run/systemd/system` exist; reads the first 20 bytes (the ELF header) of the interpreter's executable when the build architecture is not otherwise known; reads the checkout's `.git/HEAD` and the loose ref it points to, or the `packed-refs` line when there is no loose ref (a ref under `refs/` of at most 200 characters that holds no character the validator refuses in any value nor a space, `~`, `^`, `:`, `?`, `*`, `[` or `\`, and not one with `..` or `@{`, a part that is empty, starts with `.`, or ends with `.lock` or `.`, a byte that is not UTF-8, or, when the file system encoding is not UTF-8, a character that is not ASCII), the head and base SHAs from the GitHub Actions event file, and a fixed allowlist of environment variables; and runs `node -p`, looked for in the `PATH` directories only (on Windows only as `node.exe`, whatever `PATHEXT` lists, so a `node.bat` or `node.com` earlier on `PATH` does not run in its place, without `shutil.which` and so with any CPython version, and not in the current directory, where `shutil.which` looks first, unless `PATH` names it, nor in the one `os.defpath` lists first when `PATH` is unset, and an empty `PATH` entry names no directory there), with an empty environment (on Windows, `SystemRoot` only), reading at most 4 KiB of its output and stopping it, with its process group, after 30 seconds. A descendant of `node` that starts a session of its own leaves that group and can keep the read waiting until it exits or the job's 10-minute timeout ends it. It reads a file only when it is a regular file, and opens it so that a FIFO cannot block the probe. Its docstring keeps the same list. It never iterates the environment, uses no network, never escalates privileges, changes no system state (it does not touch `binfmt_misc`, which can trigger an automount, and decides cgroup delegation from permission bits without writing), and writes only its output file and the optional summary. It records no host name, account name, home directory, network address, serial number, or other environment variable. It refuses `--stage-reason` and `--stage-evidence` values shaped like a token or a private key, and does not write a record that holds such a value. On systems other than Linux it records the OS family but not its version, so a real macOS profile that declares the macOS version its status needs fails against its run as `profile-unverified` until the collector reads that version; it also records the user and group IDs on FreeBSD (`os.getresuid`) and macOS (`os.geteuid` and `os.getuid`), the Python and Node runtimes, and the run, and leaves the layer kind and the isolation facts `unknown`. macOS has no `getresuid`, so there the saved user ID is inferred from the effective one, which `execve(2)` saves. On any other system, such as Cygwin or MSYS2, user ID 0 does not name the Windows administrator, so the user IDs are `unknown`. On Windows `superuser` is `unknown`, because the collector does not read the token, and the real and saved user IDs are `not-applicable`. + +It describes its own process. It does not see the privileges or environment of other steps in the same job; a job that runs later stages passes their status, evidence, and escalation with the `--stage`, `--stage-reason`, `--stage-evidence`, and `--stage-escalation` options. The statuses are recorded as given and the escalation as `declared`, so a stage's status is only as good as the step that sets it (see "Adding a lane"). It does not assume a guest can see its physical host: on a VM, the host layer's facts are `unknown`. With no container marker and the initial PID namespace, or under WSL2 the distribution's own, it takes the root file system's os-release and PID 1 to be those of the layer that runs the kernel, and records them as `inferred`, because a chroot or a container that shares that namespace without a marker looks the same. + +The kernel release comes from `/proc/sys/kernel/osrelease`, or, when that file cannot be read, from `uname` as `inferred`, because personality changes and qemu-user can rewrite what `uname` reports. WSL2 is recognised from that release string only, by `-microsoft-standard` or `WSL2` in it; a Microsoft kernel without either is recorded as an unknown kind, a custom kernel that does not name Microsoft looks like a Hyper-V VM, and under WSL2 the Windows layer is inferred to be the outermost host. The WSL environment variables are not read. A container's engine is named only when every marker that names one agrees; markers that disagree, such as `/.dockerenv` with `/run/.containerenv`, leave it `unknown` with all of them as evidence. + +## Profiles, workflows, and evidence + +- A profile's `evidence_sources` name a workflow file under `.github/workflows/` and a job in its `jobs:` mapping; for a real profile the validator checks that both exist and that there is at least one. It reads the workflow as text, not as YAML, with lines that end at `\n`, `\r\n`, or `\r`, as in YAML, but not at U+0085, U+2028, or U+2029: a job is a key of the block mapping under an unindented `jobs:` line (or `"jobs":`, `'jobs':`, or `jobs :`, with nothing after the colon but blanks and a comment after a blank), at the indentation of the mapping's first key, plain or quoted, with a colon followed by a space, a tab, or the end of the line. A `jobs` mapping written in flow style names no job, so such a profile fails with `evidence-source`. A synthetic profile is recorded by no job, so its sources are not checked. +- A result names its profile in `profile_id`. With `--profile`, the validator checks that the IDs match, that a real profile's record was measured as a GitHub Actions run, that its workflow and job are one of the profile's evidence sources (the workflow named by `run.ci.workflow_ref`, which a GitHub Actions record must carry with `run.ci.repository`; the collector records the ref, the workflow ref, and the commit of a branch checked out by its name for any ref git allows in a branch or tag name unless it holds a character the validator refuses in any value or a byte that is not UTF-8, or, when the file system encoding is not UTF-8, a character that is not ASCII, or, for the commit, a part that ends with `.`, up to the schema's 200 characters for the ref and for the whole workflow ref, so a manual dispatch from a ref it cannot record fails `ci-evidence` and `profile-link`, and one whose commit alone it cannot record fails `ci-evidence`), and that every fact the profile declares, in its layers, runtimes, backends (matched by ID), accelerator inventory, and runner, is matched by an `observed` or `inferred` value in the result. A fact the profile leaves `unknown` is not compared, so a profile declares only what its job can observe; a fact it declares `not-applicable` must not be measured in the result. Facts inside `extensions` are not compared. +- A profile may declare the runner its job uses in `runner` (`runner_environment`, `runner_os`, `runner_arch`, `image_os`), compared with the result's `run.ci` facts. `linux-x64-gha` declares a GitHub-hosted Linux X64 runner on the `ubuntu24` image, so a self-hosted record does not satisfy it. +- The CI job validates its record with `--require-real`, `--profile`, and `--dependency-strategy`, and uploads it as an artifact whether or not validation passed (see [TESTING.md](TESTING.md) "Continuous integration"). The collector does not write a record that holds a secret-shaped value, so the validator's `secret-like-value` check is a second check, not the gate before upload. + +### Commits + +`run.revision` keeps four commit IDs apart: `tested_sha`, the commit the checkout holds (`.git/HEAD`); `event_sha`, `GITHUB_SHA`, which for a `pull_request` run is GitHub's temporary merge commit; and `pr_head_sha` and `pr_base_sha`, the pull request's head and base, read from the event file. Only these two values are taken from the event payload. The run URL is inferred from `GITHUB_SERVER_URL`, `GITHUB_REPOSITORY`, and `GITHUB_RUN_ID`. + +### Current lanes and fixtures + +Each profile's `support` field copies its status from the lane table; this list names the lane only. + +| Document | Kind | Lane named in `support` | Run by | +|---|---|---|---| +| `profiles/linux-x64-gha.json` | real profile, required stage `detect` | Linux x86-64 glibc | `compatibility` job in `.github/workflows/ci.yml` | +| `fixtures/linux-x64-vm.*` | synthetic, CPU only | Linux x86-64 glibc (`not-assessed`, as a synthetic profile) | none | +| `fixtures/windows-arm64-native.*` | synthetic, install failed, QNN NPU listed | Windows on Arm, native (`win_arm64`) | none | +| `fixtures/windows-arm64-wsl2-podman.*` | synthetic, rootless Podman with a non-root user | Windows on Arm with WSL2 (Ubuntu arm64) | none | +| `fixtures/vm-podman-quadlet-systemd.result.json` | synthetic, systemd in the container and on the VM | no profile | none | +| `fixtures/kubernetes-arm64-cuda.*` | synthetic, CUDA as `accelerator_backend`, approved device exception | Linux aarch64 glibc (`not-assessed`, as a synthetic profile) | none | + +## Adding a lane + +A later change that adds an OS, ABI, accelerator, or runtime lane: + +1. Adds a profile under `tests/compatibility/profiles/` that declares only what its job can observe, copies its support status from the lane table (or uses `not-assessed` for a lane the table does not list, or for an interpreter other than the lock's), names its `gpu` or `npu` backend in `accelerator_backend` if it is an accelerator lane, declares its runner, and names the job in `evidence_sources`. +2. Adds the job, which runs the collector with `--profile-id` and passes the stages its other steps ran with the `--stage*` options, citing their logs or artifacts and their privilege escalation, then runs the validator with `--require-real`, `--profile`, and `--dependency-strategy`. The collector step runs with `if: ${{ !cancelled() }}` after the workload steps and takes each status from that step's `steps..outcome`: `success` gives `passed`, `failure` gives `failed`, and `skipped` gives `blocked` or `not-run` with the reason, each citing that step's log. +3. Adds whatever probe the lane needs that the collector does not have, such as Windows or macOS facts (token elevation and privileges, the layer kind), engine-side container facts, or accelerator inventory, without per-technology validators. The `--stage*` options cannot record a stage's `compute`, and the collector lists only the `cpu` backend, so an accelerator lane also needs a way to record its backend and each stage's requested and effective backend before its profile can pass. Values outside the documented lists warn, and the rules that need a documented value skip them, until they are added to the lists; data without a field goes in `x-` extensions. A change that removes or narrows a field increments `schema_version`. The probe tests describe made-up Linux machines with the interpreter facts fixed to match each one, so they do not depend on the host's interpreter, but so far they have run only on Linux x86-64. +4. Changes a lane's support status only by editing [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md) with the evidence. + +## What a passing job means + +A passing compatibility job shows, for that commit and runner, that the collector ran there and wrote a record that is structurally valid, breaks no rule above, measures every fact the profile declares, including a GitHub-hosted runner, and passes `detect`; that the committed profiles validate against the lane table; and that the validator's and collector's tests passed, including the documents that must fail. + +It does not show that any other OS, architecture, Python or Node version, or accelerator works; that the application installs, starts, or behaves correctly on this lane (those stages are `not-run`, and `overall` is `incomplete`); that no step of the job escalated privileges (GitHub-hosted runners provide passwordless `sudo`, which this job does not use and the collector does not test, and membership in the `docker` group, which would also be root-equivalent, is not checked; `detect`'s `none` is declared by the collector); or that emulation was ruled out beyond the limits under "CPU execution". The job is not a required check, and a green run is not a review or an approval. diff --git a/docs/CODE-REVIEW.md b/docs/CODE-REVIEW.md index 418b17e..c031b01 100644 --- a/docs/CODE-REVIEW.md +++ b/docs/CODE-REVIEW.md @@ -7,7 +7,7 @@ It is not a security policy (see [SECURITY-ARCHITECTURE.md](SECURITY-ARCHITECTUR ## Status - In effect for every code change, starting at HEAD 8ec5623 (committed 2026-09-21; the repository facts in this document were verified on 2026-10-08). The procedure depends on nothing that is Planned. (Updated 2026-10-10: the Frontend boundary checks were re-checked at 14b9109; unchanged through 3fe4d1a.) -- Tooling: `.github/workflows/ci.yml` is the only workflow in this repository's files. It runs the baseline backend, frontend, and Playwright checks, plus the pytest and Vitest unit test jobs, on pull requests to `main` and pushes to `main`, on the checked lane only; [TESTING.md](TESTING.md) "Continuous integration" lists its jobs and what it does not cover. Its runs are evidence to read (see "Independence and authority" and step 8). GitHub-side integrations, such as review bots and required status checks, are configured outside the repository's files; whether one runs depends on the connected service and its settings, and its output is evidence to read. `.github/` also holds `copilot-instructions.md`, a bridge file that points to [CONTEXT.md](CONTEXT.md), and the advisory issue forms and pull request template described in [CONTEXT.md](CONTEXT.md) "GitHub issues and pull requests"; they are writing aids, not review tooling or evidence. Adding another workflow, or extending `ci.yml` beyond the checked lane, is a separate, explicitly requested change. (Updated 2026-10-10.) +- Tooling: `.github/workflows/ci.yml` is the only workflow in this repository's files. It runs the baseline backend, frontend, and Playwright checks, plus the pytest and Vitest unit test jobs and the compatibility job of [CI-MATRIX.md](CI-MATRIX.md), on pull requests to `main` and pushes to `main`, on the checked lane only; [TESTING.md](TESTING.md) "Continuous integration" lists its jobs and what it does not cover. Its runs are evidence to read (see "Independence and authority" and step 8). GitHub-side integrations, such as review bots and required status checks, are configured outside the repository's files; whether one runs depends on the connected service and its settings, and its output is evidence to read. `.github/` also holds `copilot-instructions.md`, a bridge file that points to [CONTEXT.md](CONTEXT.md), and the advisory issue forms and pull request template described in [CONTEXT.md](CONTEXT.md) "GitHub issues and pull requests"; they are writing aids, not review tooling or evidence. Adding another workflow, or extending `ci.yml` beyond the checked lane, is a separate, explicitly requested change. (Updated 2026-10-11.) - Verification baseline: the automated tests that exist are listed in [TESTING.md](TESTING.md) under "Tests that exist today". They were not executed in this documentation pass. "Exists" and "executed" stay distinct throughout this document. - The checks under "Boundary-specific checks" for Realtime, Task, and Runtime cover Planned areas. They apply to a change that introduces such code; they never imply the feature exists. @@ -156,7 +156,7 @@ A change that touches any of the following also goes through [SECURITY-REVIEW.md - External content handling: web pages, uploaded files, retrieved text, or tool output that reaches a model or a user. - Logging and audit: what is written to logs, Task records, or tool run summaries. - i18n locale input handling: cookies, `Accept-Language`, preference values, `` ([I18N.md](I18N.md)). -- CI workflows: any file under `.github/workflows/`, including its trigger events, `permissions`, secrets, `runs-on` labels, and action references. +- CI workflows: any file under `.github/workflows/`, including its trigger events, `permissions`, secrets, `runs-on` labels, and action references, and the scripts and data a workflow runs and publishes the output of: `scripts/ci/` and `tests/compatibility/`. (Updated 2026-10-11.) The code review report states which triggers matched and whether the security review was done or is pending. Security findings use the finding format below plus the `SEC-*` IDs they map to, as [SECURITY-REVIEW.md](SECURITY-REVIEW.md) specifies. diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index 726bfdb..d72d415 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -34,6 +34,7 @@ Implemented today, each with tests in the repository: the health APIs, the ASGI - [`docs/STATE-SCHEMA.md`](STATE-SCHEMA.md) describes conceptual collaboration and runtime state shapes. It is not a database schema. - [`docs/TESTING.md`](TESTING.md) describes testing strategy, the checks that exist today, and the verification still required. - [`docs/DEPENDENCY-STRATEGY.md`](DEPENDENCY-STRATEGY.md) records the applied dependency baseline, compatibility exceptions and residual advisories, platform and accelerator lanes, and the Python 3.15 Limited API/`abi3t` qualification and performance plan. It builds on the existing free-threading direction; the baseline update does not qualify 3.15/3.15t. +- [`docs/CI-MATRIX.md`](CI-MATRIX.md) defines the compatibility model: how a lane is declared as a profile, how a run is recorded and validated (layers, privilege, runtimes, compute backends, stages), and what a passing compatibility job means. It records no support status; the lane table in DEPENDENCY-STRATEGY.md does. - [`docs/SECURITY-ARCHITECTURE.md`](SECURITY-ARCHITECTURE.md) defines trust boundaries and the `SEC-*` security requirements, with the current implementation facts. It is the only place those requirement IDs are defined. - [`docs/CODE-REVIEW.md`](CODE-REVIEW.md) defines how a coding agent independently reviews an actual code change and reports findings. - [`docs/SECURITY-REVIEW.md`](SECURITY-REVIEW.md) defines the security review procedure for changed code, reusing the `SEC-*` IDs. @@ -52,6 +53,7 @@ Read this file first. Then read only the documents that match the work; do not r | Work that touches collaboration, Task, Artifact, Knowledge, or presentation state | [STATE-SCHEMA.md](STATE-SCHEMA.md) | | Before claiming anything is tested, and when adding tests | [TESTING.md](TESTING.md) | | Dependency, interpreter, native wheel/ABI, platform or accelerator, or packaging-tool updates | [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md), then [TESTING.md](TESTING.md) | +| Compatibility profiles, environment records, or a new OS, CPU, runtime, container, or accelerator lane in CI | [CI-MATRIX.md](CI-MATRIX.md), then [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md) and [TESTING.md](TESTING.md) | | Changes to `.github/workflows/` or other CI configuration | [TESTING.md](TESTING.md) "Continuous integration", then [SECURITY-ARCHITECTURE.md](SECURITY-ARCHITECTURE.md) and [SECURITY-REVIEW.md](SECURITY-REVIEW.md) | | Reviewing a code change (yours or another agent's) | [CODE-REVIEW.md](CODE-REVIEW.md) | | Creating or editing a GitHub issue or pull request at the user's explicit request or a supported delegation | [GitHub issues and pull requests](#github-issues-and-pull-requests) below, then the matching template in [`.github/ISSUE_TEMPLATE/`](../.github/ISSUE_TEMPLATE/) or [`.github/pull_request_template.md`](../.github/pull_request_template.md) | diff --git a/docs/DEPENDENCY-STRATEGY.md b/docs/DEPENDENCY-STRATEGY.md index 75596d2..9a9eeae 100644 --- a/docs/DEPENDENCY-STRATEGY.md +++ b/docs/DEPENDENCY-STRATEGY.md @@ -176,9 +176,13 @@ Statuses used for the backend lock, installed binary-only: - **Statically resolvable**: every pin has a hashed wheel for the platform, and no dependency required only on that platform is missing from the lock. Not executed. - **Not supported**, **not binary-installable** or **incomplete**: a required wheel or platform-only dependency is missing, so the current lock cannot produce a working environment there without a new resolution or a source build (Windows x64 installs without `tzdata`; the other cases fail to install). +This table is the only record of a lane's support status. Its statuses are for the backend lock, which is resolved for CPython 3.12 with the GIL. Compatibility profiles in `tests/compatibility/` copy a lane's status and name from it, a profile for another interpreter, such as 3.15 or 3.15t, names the lane with the status `not-assessed`, and `scripts/ci/validate_compatibility.py` fails a profile whose OS family, architecture, C library, or WSL2 layer is not the lane's, or whose declared versions do not meet the bounds of the status it claims, and with `--dependency-strategy` one that disagrees with the Lane and Backend lock columns, so renaming a lane or changing its status word changes what a profile may claim ([CI-MATRIX.md](CI-MATRIX.md#stage-status-and-support-status)). No compatibility run changes a status here; a status changes only by editing this table with its evidence. + +A status can hold for some versions only. Its cell then writes each bound as ` or newer` or `or older`, and the validator keeps the same bounds, which a test compares with the cells. Linux x86-64 glibc is Checked for glibc 2.39 or newer and CPython 3.12.13 or newer, the lowest versions its recorded installs used (those installs used glibc 2.39 and 2.41 and CPython 3.12.13 and 3.12.15, and package tests also passed on 3.12.14); other glibc and CPython 3.12 releases at or above those bounds are taken to keep binary compatibility, which no install here has tested. A profile below a bound, such as one on RHEL 9 (glibc 2.34), on CPython 3.12.12, or on a pre-release, names the lane with `not-assessed`. macOS Apple Silicon is statically resolvable for macOS 15 or newer and not supported for macOS 14 or older. The distribution is not part of a status: a profile on another distribution with the same glibc and CPython versions copies the row's status, so a difference that only the distribution makes, such as its other system libraries, is outside the evidence. A statically resolvable Linux row has no glibc bound, although its wheels' manylinux tags set one. + | Lane | Representative devices | Backend lock, binary-only | Frontend npm lock | Playwright browsers | Accelerator | |---|---|---|---|---|---| -| Linux x86-64 glibc | Existing baseline | Checked (author-reported hashed-lock install with the pinned toolchain on glibc 2.41; supplemental fix-session hashed-lock install with uv 0.11.32 and CPython 3.12.13 on glibc 2.39; follow-up pinned-toolchain hashed-lock install with uv 0.12.24 and CPython 3.12.15 on glibc 2.39 at 53ac8c5; see [the checks above](#applied-baseline-and-verification)) | OK | Supported | CUDA optional (none used) | +| Linux x86-64 glibc | Existing baseline | Checked for glibc 2.39 or newer and CPython 3.12.13 or newer (author-reported hashed-lock install with the pinned toolchain on glibc 2.41; supplemental fix-session hashed-lock install with uv 0.11.32 and CPython 3.12.13 on glibc 2.39; follow-up pinned-toolchain hashed-lock install with uv 0.12.24 and CPython 3.12.15 on glibc 2.39 at 53ac8c5; see [the checks above](#applied-baseline-and-verification)) | OK | Supported | CUDA optional (none used) | | Linux aarch64 glibc | NVIDIA DGX Spark (GB10: 20-core Arm CPU and Blackwell GPU; DGX OS 7.5.0, Ubuntu-based; CUDA 13.0.2 per the DGX Spark release notes), Arm servers, Linux on Snapdragon X2 (Debian support targeted for the end of 2026, Ubuntu certification in H1 2027), WSL2 on Windows on Arm | Statically resolvable: all 50 pins have a hashed wheel installable on manylinux aarch64 (11 native aarch64 wheels, 39 pure-Python wheels), and no platform-only dependency is missing | `linux-arm64-gnu` binaries present (`@next/swc`, `lightningcss`, `@tailwindcss/oxide`, `sharp`, `unrs-resolver`) | Chromium on Ubuntu 22.04/24.04/26.04 and Debian 12/13 arm64 (the Google Chrome channel is not available on Linux arm64) | CUDA on DGX Spark only | | Windows on Arm, native (`win_arm64`) | NVIDIA RTX Spark PCs (Arm Grace CPU and Blackwell RTX GPU on Windows on Arm; the CUDA Toolkit 13.4 developer preview adds Windows on Arm; OEM systems announced for fall 2026), Qualcomm Snapdragon X / X2 (no CUDA; CPU/GPU/NPU through QNN, ONNX Runtime and Windows ML) | Not supported: `tzdata`, which Django and psycopg require on win32, is absent from the Linux-resolved lock, and `autobahn`, `cryptography` and `psycopg-binary` have no `win_arm64` wheels | `win32-arm64-msvc` binaries present | Playwright ships only x64 browsers on Windows; they run under Prism emulation ([microsoft/playwright#40202](https://github.com/microsoft/playwright/issues/40202) was closed as not planned on 2026-05-12; Chrome for Testing has no win-arm64 build) | CUDA on RTX Spark only | | Windows on Arm with WSL2 (Ubuntu arm64) | Same devices | Same as Linux aarch64 glibc | OK | Linux arm64 Chromium | — | diff --git a/docs/README.en.md b/docs/README.en.md index 42f7b18..18d2866 100644 --- a/docs/README.en.md +++ b/docs/README.en.md @@ -46,6 +46,7 @@ Technical documents for AI coding agents and contributors are written in English - [STATE-SCHEMA.md](STATE-SCHEMA.md): conceptual state shapes (not a database schema). - [TESTING.md](TESTING.md): the tests that exist today and the verification still required. - [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md): the applied dependency update and its compatibility/security exceptions, plus Python 3.15 Limited API/`abi3t` wheel qualification, performance admission, and maintenance under the existing free-threading direction. Python 3.15 support remains unqualified. +- [CI-MATRIX.md](CI-MATRIX.md): the compatibility model for CI lanes, covering profiles, run records, and what the validator checks. It records no support status. - [SECURITY-ARCHITECTURE.md](SECURITY-ARCHITECTURE.md): trust boundaries and the `SEC-*` security requirements. - [CODE-REVIEW.md](CODE-REVIEW.md), [SECURITY-REVIEW.md](SECURITY-REVIEW.md): independent code review and security review procedures for code changes. - [INTERACTION-INTERFACES.md](INTERACTION-INTERFACES.md), [WEBMCP.md](WEBMCP.md), [I18N.md](I18N.md): the not-yet-implemented interaction architecture, the WebMCP (experimental technology) contract, and the frontend i18n direction. diff --git a/docs/README.ko.md b/docs/README.ko.md index c1531f0..179e54e 100644 --- a/docs/README.ko.md +++ b/docs/README.ko.md @@ -46,6 +46,7 @@ AI 코딩 에이전트와 기여자를 위한 기술 문서는 영어로 작성 - [STATE-SCHEMA.md](STATE-SCHEMA.md): 개념적 상태 모양(DB schema 아님). - [TESTING.md](TESTING.md): 현재 존재하는 테스트와 앞으로 필요한 검증. - [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md): 적용한 의존성 업데이트와 호환성·보안 예외, 기존 free-threading 방침에 따른 Python 3.15 Limited API/`abi3t` wheel 대응, 성능 검증 및 유지보수 전략. Python 3.15 지원은 아직 미검증입니다. +- [CI-MATRIX.md](CI-MATRIX.md): CI lane의 호환성 모델(profile, 실행 기록, validator가 검사하는 내용). 지원 상태는 기록하지 않습니다. - [SECURITY-ARCHITECTURE.md](SECURITY-ARCHITECTURE.md): 신뢰 경계와 `SEC-*` 보안 요구사항. - [CODE-REVIEW.md](CODE-REVIEW.md), [SECURITY-REVIEW.md](SECURITY-REVIEW.md): 코드 변경의 독립 리뷰와 보안 리뷰 절차. - [INTERACTION-INTERFACES.md](INTERACTION-INTERFACES.md), [WEBMCP.md](WEBMCP.md), [I18N.md](I18N.md): 아직 구현되지 않은 상호작용 구조, WebMCP(실험적 기술) 계약, 프런트엔드 i18n 방향. diff --git a/docs/SECURITY-ARCHITECTURE.md b/docs/SECURITY-ARCHITECTURE.md index 72e43b6..5ca8828 100644 --- a/docs/SECURITY-ARCHITECTURE.md +++ b/docs/SECURITY-ARCHITECTURE.md @@ -136,7 +136,7 @@ Verified on 2026-10-08. (Updated 2026-10-10: the `/core` tests, ASGI composition - `.gitignore` ignores `.env` and `.env.*` except example files (lines 30-33), private key files (lines 36-43), credential-shaped JSON (lines 46-53), logs (lines 64-65), `backend/db.sqlite3` (line 26), and browser profile directories (lines 83-88). - `.env.example` (lines 2-13) holds empty placeholders for `DJANGO_SECRET_KEY`, `DATABASE_URL`, and the five storage variables, plus `DJANGO_DEBUG=true`. Lines 15-20 hold two empty, non-secret, test-only Playwright variables, `PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH` (line 18) and `PLAYWRIGHT_NEXT_SERVER` (line 20), with comments stating that Playwright does not load the file. (Updated 2026-10-10.) - The ignore rules only keep untracked files out of a commit. They do not apply to a file that is already tracked or that is added with `git add -f`, and they do not catch a secret written into source code, documentation, tests, or configuration, or one already in the history. -- No Git hook or secret scanner is configured in the repository's files. The one workflow, `.github/workflows/ci.yml`, runs build and test checks and scans nothing for secrets (see "CI workflow" below); `.github/` also holds `copilot-instructions.md` and advisory issue and pull request templates, none of which runs a check. (Updated 2026-10-10.) [SECURITY-REVIEW.md](SECURITY-REVIEW.md) step 7 defines a keyword check of a change's added lines, which a reviewer runs; nothing runs or enforces it automatically. +- No Git hook or secret scanner is configured in the repository's files. The one workflow, `.github/workflows/ci.yml`, runs build and test checks and scans nothing for secrets, apart from the compatibility collector's and validator's checks of their own records (see "CI workflow" below); `.github/` also holds `copilot-instructions.md` and advisory issue and pull request templates, none of which runs a check. (Updated 2026-10-11.) [SECURITY-REVIEW.md](SECURITY-REVIEW.md) step 7 defines a keyword check of a change's added lines, which a reviewer runs; nothing runs or enforces it automatically. ### Frontend @@ -147,14 +147,15 @@ Verified on 2026-10-08. (Updated 2026-10-10: the `/core` tests, ASGI composition ### CI workflow (`.github/workflows/ci.yml`) -Added 2026-10-10. The workflow runs the baseline checks listed in [TESTING.md](TESTING.md) "Continuous integration". +Added 2026-10-10; the compatibility job was added 2026-10-11. The workflow runs the baseline checks listed in [TESTING.md](TESTING.md) "Continuous integration". - Triggers: `push` and `pull_request` for `main`, and `workflow_dispatch` (lines 8-13). There is no `pull_request_target` or `workflow_run` trigger, so pull request code, including a fork's, runs with the token GitHub gives that pull request, which for a fork is read-only and comes without the repository's secrets. - The workflow token is limited to `contents: read` (lines 15-16). No job references a secret. -- Every `actions/checkout` step sets `persist-credentials: false`, so the token is not left in the runner's Git configuration for later steps (lines 35-37, 71-73, 106-108, 155-157, 200-202). -- Every action is pinned to a full commit SHA, with its version as a comment (lines 35, 38, 71, 74, 106, 109, 121, 138, 155, 158, 200, 203). Their freshness and provenance review is in [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md#security-floors-and-release-freshness). -- Every job runs on a GitHub-hosted `ubuntu-24.04` runner (lines 32, 64, 95, 149, 193); no self-hosted runner is referenced. A job installs tools from the network at run time: uv through `astral-sh/setup-uv`, which checks it against the action's known SHA-256, CPython through `uv venv --managed-python`, the hash-locked Python packages (the backend lock, or in the pytest job the integrated test lock; uv checks each file it downloads against the lock's hashes, so a hash is only as trustworthy as the reviewed lock line), Node through `actions/setup-node`, npm 11.21.0 by version without a hash, the npm lock's packages, and Chromium with its system packages. The pytest and Vitest jobs (added 2026-10-10) run only repository code, the locked test tools, two scripts in `scripts/ci/`, and the lock check's tests: `check_test_lock.py` reads the locks, the requirement inputs, and the installed packages' metadata (`importlib.metadata`) with the standard library and `packaging` from the test lock, and its regression tests in `scripts/ci/tests/` (standard library `unittest`) run it in a subprocess on copies of the four files in temporary directories, or call its functions in-process on made-up inputs. The pytest job sets `UV_NO_CONFIG` (line 153), so its uv commands ignore a `uv.toml` or a `pyproject.toml` `[tool.uv]` table that a pull request adds, which could otherwise change the index, the packages, or the CPython download uv installs; the backend and E2E jobs do not set it, so such a file can still change what they install. Its `astral-sh/setup-uv` step is given `python-version` (line 163); without it, setup-uv runs `uv python find` in the checkout and executes the interpreter it finds, which a pull request could plant as `.venv/bin/python` (the backend and E2E jobs still do this). In the pytest job the lock check and its tests run right after the install, before any step that runs Python from the checkout, and as scripts with `python -I`, so the checkout root, `PYTHONPATH`, `scripts/ci/`, and `scripts/ci/tests/` are not on their import path, and a module or `.dist-info` directory placed there cannot stand in for an installed one. The check catches inputs and locks that disagree, not a pull request that changes the workflow, the check, or its tests, which then run as changed, nor one whose locks add or swap in a malicious package: the check runs inside the environment installed from those locks, and a package's `.pth` file runs at interpreter start even under `python -I`. Those need review in the diff; `check_test_parity.py` imports Django from the test lock, builds the Django test suite (importing every test module), and runs `pytest --collect-only`. Their tests use SQLite, a fake S3 client, and mocked HTTP and alert modules, and reach no external service. -- The E2E jobs upload `frontend/test-results/` as a run artifact kept for 7 days (lines 138-143). Logs and artifacts of runs in this public repository can be read by people outside the project, so a workflow step must not print secrets or personal data. +- Every `actions/checkout` step sets `persist-credentials: false`, so the token is not left in the runner's Git configuration for later steps (lines 35-37, 71-73, 106-108, 155-157, 200-202, 228-230). +- Every action is pinned to a full commit SHA, with its version as a comment (lines 35, 38, 71, 74, 106, 109, 121, 138, 155, 158, 200, 203, 228, 231, 237, 264). Their freshness and provenance review is in [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md#security-floors-and-release-freshness). +- Every job runs on a GitHub-hosted `ubuntu-24.04` runner (lines 32, 64, 95, 149, 193, 222); no self-hosted runner is referenced. A job installs tools from the network at run time: uv through `astral-sh/setup-uv`, which checks it against the action's known SHA-256, CPython through `uv venv --managed-python`, the hash-locked Python packages (the backend lock, or in the pytest job the integrated test lock; uv checks each file it downloads against the lock's hashes, so a hash is only as trustworthy as the reviewed lock line), Node through `actions/setup-node`, npm 11.21.0 by version without a hash, the npm lock's packages, and Chromium with its system packages. The pytest and Vitest jobs (added 2026-10-10) run only repository code, the locked test tools, two scripts in `scripts/ci/`, and the lock check's tests: `check_test_lock.py` reads the locks, the requirement inputs, and the installed packages' metadata (`importlib.metadata`) with the standard library and `packaging` from the test lock, and its regression tests in `scripts/ci/tests/` (standard library `unittest`) run it in a subprocess on copies of the four files in temporary directories, or call its functions in-process on made-up inputs. The pytest job sets `UV_NO_CONFIG` (line 153), so its uv commands ignore a `uv.toml` or a `pyproject.toml` `[tool.uv]` table that a pull request adds, which could otherwise change the index, the packages, or the CPython download uv installs; the backend and E2E jobs do not set it, so such a file can still change what they install. Its `astral-sh/setup-uv` step is given `python-version` (line 163); without it, setup-uv runs `uv python find` in the checkout and executes the interpreter it finds, which a pull request could plant as `.venv/bin/python` (the backend and E2E jobs still do this). In the pytest job the lock check and its tests run right after the install, before any step that runs Python from the checkout, and as scripts with `python -I`, so the checkout root, `PYTHONPATH`, `scripts/ci/`, and `scripts/ci/tests/` are not on their import path, and a module or `.dist-info` directory placed there cannot stand in for an installed one. The check catches inputs and locks that disagree, not a pull request that changes the workflow, the check, or its tests, which then run as changed, nor one whose locks add or swap in a malicious package: the check runs inside the environment installed from those locks, and a package's `.pth` file runs at interpreter start even under `python -I`. Those need review in the diff; `check_test_parity.py` imports Django from the test lock, builds the Django test suite (importing every test module), and runs `pytest --collect-only`. Their tests use SQLite, a fake S3 client, and mocked HTTP and alert modules, and reach no external service. +- The compatibility job (added 2026-10-11, lines 217-271) downloads uv and CPython 3.12.15 as the pytest job does, and Node from `.nvmrc` through `actions/setup-node` as the frontend jobs do, with `UV_NO_CONFIG` set (line 226) and `python-version` given to `astral-sh/setup-uv` (line 236), installs no package, and runs only the standard-library scripts `scripts/ci/collect_environment.py` and `scripts/ci/validate_compatibility.py` and their tests in `scripts/ci/tests/`, each with `python -I`. The collector reads `/proc`, `/sys`, the OS release file, a few files under `/run`, the interpreter's ELF header, the checkout's `.git/HEAD` and the ref it points to, a fixed allowlist of environment variables (never the whole environment), and only the pull request's head and base SHAs from the event file, and runs `node -p`, found in the `PATH` directories only (on Windows only as `node.exe`, whatever `PATHEXT` lists, and not in the current directory unless `PATH` names it, also when `PATH` is unset), with an empty environment (on Windows, `SystemRoot` only), reading at most 4 KiB of its output and stopping it, with its process group, after 30 seconds (a descendant that starts a session of its own can hold the read until the job times out; [CI-MATRIX.md](CI-MATRIX.md) "Collector" lists every source); it reads only regular files, so a FIFO cannot block it; it records no host name, account name, home directory, or network address, uses no network, and does not use the runner's `sudo`. Its record and summary describe the runner, not the repository's users. The collector refuses stage reasons and evidence shaped like common tokens or private keys and does not write a record that holds such a value, and the validator rejects records with control characters or such values (`secret-like-value`); in the lines both scripts print, their command-line errors included, and in the validator's summary, `##[` becomes `## [`, `::` becomes `: :`, and a secret-shaped value is not shown, so neither a record nor a file name can start a workflow command or show such a value in the log; these checks cover compatibility records only, scan nothing else, and do not stop a pull request that changes the collector, the validator, or their tests, which then run as changed, the same limit as the lock check above. +- The E2E jobs upload `frontend/test-results/` as a run artifact kept for 7 days (lines 138-143), and the compatibility job uploads its record whether or not validation passed, kept for 90 days (lines 264-271). Logs and artifacts of runs in this public repository can be read by people outside the project, so a workflow step must not print secrets or personal data. ## Requirements diff --git a/docs/SECURITY-REVIEW.md b/docs/SECURITY-REVIEW.md index 2ccd64c..fba4bce 100644 --- a/docs/SECURITY-REVIEW.md +++ b/docs/SECURITY-REVIEW.md @@ -6,7 +6,7 @@ It is not a security policy and defines no requirement: the meaning and status o ## Status -- In effect for every change that matches a trigger below or a run condition under "When this review runs", from HEAD 8ec5623 (committed 2026-09-21; the repository facts in this document were verified on 2026-10-08). The procedure itself depends on nothing that is Planned. (Updated 2026-10-10: the step 1 file table was re-checked at 14b9109 (unchanged through 3fe4d1a) and now lists `frontend/src/proxy.ts`, the lockfiles, and the runtime version files. Later the same day, CI workflows were added to the triggers and to the step 1 file table.) +- In effect for every change that matches a trigger below or a run condition under "When this review runs", from HEAD 8ec5623 (committed 2026-09-21; the repository facts in this document were verified on 2026-10-08). The procedure itself depends on nothing that is Planned. (Updated 2026-10-10: the step 1 file table was re-checked at 14b9109 (unchanged through 3fe4d1a) and now lists `frontend/src/proxy.ts`, the lockfiles, and the runtime version files. Later the same day, CI workflows were added to the triggers and to the step 1 file table. On 2026-10-11, `scripts/ci/` and `tests/compatibility/` were added to the CI trigger and to that table row, and step 1 gained the check of compatibility policy exceptions.) - Implemented today, and therefore reviewable as behavior: the health endpoints, the ASGI composition, `DATABASE_URL` parsing with SQLite fallback, and the S3-compatible storage backend. Their security facts, with file and line evidence, are in [SECURITY-ARCHITECTURE.md](SECURITY-ARCHITECTURE.md) "Current implementation security facts". - Not implemented: authentication, authorization, scope checks, sharing, approval, audit, RAG, model calls, Tasks, runtimes beyond a lazy Playwright factory, and WebMCP. Most `SEC-*` requirements are Planned; "How Planned requirements are reviewed" below says what that means for a review. - WebMCP is Experimental: an external draft with no default-on browser support as of 2026-10-08 ([WEBMCP.md](WEBMCP.md)). In the AGENT area, `SEC-AGENT-001` is Planned and `SEC-AGENT-002` to `SEC-AGENT-005` are Planned/Experimental. @@ -36,7 +36,7 @@ Triggers are the ones listed in [CODE-REVIEW.md](CODE-REVIEW.md) step 9. Area na | External content handling (web pages, uploaded files, retrieved text, tool output reaching a model or a user) | INJ, `SEC-RAG-002` | `SEC-SECRET-003`, `SEC-TOOL-002` | | Logging and audit | `SEC-SECRET-003`, `SEC-SECRET-004`, `SEC-TOOL-002` | `SEC-FILE-005` | | i18n locale input handling (cookies, `Accept-Language`, preference values, ``) | `SEC-INJ-001`, `SEC-AGENT-002` | AUTH, when a preference is stored on the account | -| CI workflows (`.github/workflows/*`: trigger events, `permissions`, secrets, `runs-on` labels, action references) | `SEC-SECRET-001` | `SEC-SECRET-003`, `SEC-TOOL-001` | +| CI workflows (`.github/workflows/*`: trigger events, `permissions`, secrets, `runs-on` labels, action references), and the scripts and data they run and publish the output of (`scripts/ci/*`, `tests/compatibility/*`) | `SEC-SECRET-001` | `SEC-SECRET-003`, `SEC-TOOL-001` | ### How Planned requirements are reviewed @@ -63,7 +63,7 @@ Map every file in the change to the boundaries in [SECURITY-ARCHITECTURE.md](SEC | `backend/agent/runtime/`, `backend/agent/llm/`, `backend/agent/rag/`, `backend/agent/orchestration/`, `backend/agent/tools/` | Runtimes, and model to content (docstring-only packages or a lazy factory today) | | `frontend/src/lib/api.ts`, `frontend/next.config.ts`, `frontend/src/proxy.ts`, `frontend/src/components/`, `frontend/src/app/` | Browser to control plane (the client side of it; `next.config.ts` and `src/proxy.ts` run in the Next.js server, which forwards `/core/*` and `/agent/*` to the backend and redirects UI paths, and authorizes nothing) | | `.env.example`, `.gitignore`, `backend/requirements.txt`, `backend/locks/*`, `frontend/package.json`, `frontend/package-lock.json`, `.nvmrc`, `.python-version` | No boundary: the "Secrets hygiene" implementation facts, plus the dependency surface | -| `.github/workflows/*` | No product boundary: repository to CI runner, where pull request code runs on the runner and with the token the workflow gives it; the "CI workflow" implementation facts | +| `.github/workflows/*`, `scripts/ci/*`, `tests/compatibility/*` | No product boundary: repository to CI runner, where pull request code runs on the runner and with the token the workflow gives it; the "CI workflow" implementation facts. The compatibility collector reads runner state and allowlisted environment variables and its record is published as a run artifact | Record: @@ -73,6 +73,8 @@ Record: For a change under `.github/workflows/`, also record the trigger events (`pull_request_target` and `workflow_run` run with the base repository's token and secrets), the `permissions` each job gets, every secret referenced, every `runs-on` label (a self-hosted runner runs pull request code on that machine), and whether each action is pinned to a full commit SHA. +For a change to a profile under `tests/compatibility/profiles/` that adds or changes a policy exception, check that its approval reference points to a decision by someone with authority over the repository, such as a maintainer's review or comment, and not to the change itself; that its scope names only the layers, stages and escalation methods, capabilities, or devices the lane needs; and that its end condition can be checked. The validator checks only that an approval is recorded, and its `policy-exception-applied` warning in the job log shows each use. A profile whose support status or lane changes without the matching edit to the lane table in [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md#platform-and-accelerator-lanes) is a finding. + A new file under `backend/agent/fastapi/routes/` or a new `path()` in `backend/core/urls.py` is always a new boundary. A dependency added to `requirements.txt` or `package.json` widens the trust surface and needs the explicit approval that [CONTEXT.md](CONTEXT.md) requires, cited in the change. A change to `backend/locks/*` or `frontend/package-lock.json` needs the same cited approval, because [CONTEXT.md](CONTEXT.md) requires it for lockfile changes. ### 2. Trace input, output, and state-change flows diff --git a/docs/TESTING.md b/docs/TESTING.md index d298ffc..2efc751 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -4,7 +4,7 @@ This document separates checks for the current runnable scaffold from future fea ## Tests that exist today -The repository contains these automated tests. Their existence is a fact; whether they pass must be shown by running them and citing the output. The backend has 23 test methods, run by both `python manage.py test core agent` and pytest (with pytest-django), which collect the same 23 test IDs. The frontend has 12 Playwright tests (one `home.spec.ts` scenario and two `proxy.spec.ts` tests, each run in four projects) and 13 Vitest unit tests in two files. The CI script `scripts/ci/check_test_lock.py` has 49 `unittest` regression tests. +The repository contains these automated tests. Their existence is a fact; whether they pass must be shown by running them and citing the output. The backend has 23 test methods, run by both `python manage.py test core agent` and pytest (with pytest-django), which collect the same 23 test IDs. The frontend has 12 Playwright tests (one `home.spec.ts` scenario and two `proxy.spec.ts` tests, each run in four projects) and 13 Vitest unit tests in two files. The CI script `scripts/ci/check_test_lock.py` has 49 `unittest` regression tests, and the compatibility tooling of [CI-MATRIX.md](CI-MATRIX.md) has 167 (95 for the validator and 72 for the environment collector). | Location | What it covers | |---|---| @@ -17,6 +17,8 @@ The repository contains these automated tests. Their existence is a fact; whethe | `frontend/tests/unit/api.test.ts` | Vitest, 5 tests. `coreApi` and `agentApi` target the same-origin `/core/` and `/agent/` prefixes with a 5000 ms timeout and are separate instances; a health request through each client, answered by a local adapter without network, goes to `/core/health/` or `/agent/health/` with the trailing slash kept. | | `frontend/tests/unit/HealthCard.test.tsx` | Vitest with React Testing Library in jsdom, 8 tests; `@/lib/api` and `sweetalert2` are mocked, so no request leaves the test. The card checks both endpoints on mount and shows each response; a failed service shows Disconnected without an alert on the initial check; a failed retry raises one SweetAlert2 alert naming each failed service (plural and singular wording), and a successful retry raises none; Retry is ignored while a check is in flight; a retry that fails after unmount raises no alert; and under React `StrictMode` a superseded check that settles later cannot overwrite the newest result. | | `scripts/ci/tests/test_check_test_lock.py` | `unittest`, 49 tests of the lock check described under "Current scaffold checks". 42 run the check with `python -I` in a subprocess on copies of the two backend locks and their requirement inputs, most of them edited, and expect a given failure message or a pass: missing, mismatched, duplicate, or unhashed pins; unexpected lock lines; a header for another target, without its command, or in uv's `--flag=value` form; test-only packages from the wrong origin; requirements that are unpinned, excluded by their specifier, dropped, or added without regenerating; unsupported input lines, including archive file names, backslash escapes, characters outside printable ASCII, and numbers too long for uv; a file that starts with a byte order mark; direct URLs; dropped, added, swapped, and unknown extras; markers that hold or are false on the target, including on a package the locks do not pin; unsupported marker forms; a `# via` annotation naming a file that is not an input of that lock; a `uv.toml` or a `pyproject.toml` with or without uv settings; and an environment that lacks a pin or holds another version. Seven call its functions in-process on made-up inputs: the metadata walk (markers evaluated for the target even when the running machine reports Windows, extras requested through other extras or after their package was first reached, and dependency problems), input markers evaluated for the target when the running machine reports Windows, the supported and unsupported marker forms, and input files that are not UTF-8 or nest a marker too deeply. | +| `scripts/ci/tests/test_compatibility_schema.py` | `unittest`, 95 tests of `scripts/ci/validate_compatibility.py`. The committed profile and the nine synthetic fixtures validate, the fixtures are marked synthetic, and together they cover the scenarios [CI-MATRIX.md](CI-MATRIX.md) lists. The schema interpreter: every keyword it applies is exercised by a case that must fail, a schema with an unsupported keyword such as `oneOf`, `anyOf`, `allOf`, `not`, or `format` is rejected, and strict loading rejects duplicate keys, `NaN`, `Infinity`, overflowing numbers, a byte order mark, invalid UTF-8, deep nesting, and oversized files, read only up to the size limit. A schema is also rejected for another dialect, `$schema`, `$id`, or `$defs` below the root, a non-string annotation or type name, a pattern that Python and ECMA-262 read differently (escaped letters such as `\d`, `\A`, `\a`, `\N`, and `\U`, `\u` escapes of surrogates, `(?` groups, `{,n}`, possessive quantifiers, and a class that opens with `]`), or a `$ref` cycle. Each rule across fields, on edited copies of the fixtures: provenance and synthetic marking (including `--require-real`), control characters (named by a path that writes a key over 128 characters once escaped as its length and the first 8 hexadecimal digits of its SHA-256, so keys of one length are told apart unless those digits agree, keeps at most 16 keys around a `~...` that no key can be mistaken for, and is built only when the walk reaches its value), lone surrogates and secret-shaped values (a sample of every shape, and of both Slack prefixes, on its own and right after an ASCII or other letter, an AWS key ID followed by an underscore or a lowercase letter and a run of 17 key characters that is not one, the shortest JSON Web Token and one a character too short, and a long run of token characters searched in linear time), a collection time that is not a valid date and time, layer nesting (including a documented technology on a layer of another kind, a WSL2 layer with another technology, a host that names one, and a guest or container whose technology is `not-applicable`), OS, CPU, and ABI contradictions (with emulated and 32-bit cases that must pass, emulation inherited by every inner layer and runtime, the ABI tag's OS and architecture also when the version is unknown, an ABI tag architecture outside the list against a listed arch fact but not an unknown or unlisted one, a CPython version read whole (pre-release and `+` forms that must pass as CPython 3.12, though only a release of 3.12.13 or newer meets the checked lane's bound, text after the version and leading zeros, which also lose the lane, and another implementation's version left unchecked), a CPython platform tag and ABI tag compared with each other when the innermost layer's OS family is unknown, or the arch fact is unknown or not a listed name (by architecture only when both name one OS), and, when the layer is known, a platform tag for another OS reported against the layer once and not compared with the ABI tag, the ABI tag's C library against the libc fact (also when one of them names another C library, and not between two other names), Windows and macOS platform tags, and a glibc or musl libc fact on Windows or macOS, named by the innermost layer, the platform tag or the SOABI alone, or on FreeBSD, named by the layer or the platform tag alone), unrecognized values as warnings only, identity (also on macOS and FreeBSD), stage fields and order, which stages may be not-applicable, the log or artifact a real stage cites, `detect` with probe errors, failed versus not-run, `overall`, required stages, CPU fallback reported as a GPU pass, a passed stage whose compute record names no backend that ran, whatever was requested (reported once, as a fallback, on a GPU request, and not for a failed stage), an accelerator profile's stages, `performance` included, requested and run on its backend, backend references, each default policy rule (including capabilities on Linux or an unknown OS family, and a workload not shown to run outside a container, which a WSL2 distribution's own PID namespace does not trigger), rootless engine and non-root user as independent rules, policy decided by measured facts only (declared values do not show compliance, an unknown privilege state fails, off Linux an unmeasured layer kind fails, and a real or saved user ID of 0 fails), exceptions that are pending, out of scope, unused, missing fields, or approved on a day that does not exist, scoped by escalation method, or offered for a state that is not measured, the warnings for an applied exception and a skipped policy, a profile that weakens a rule, support claims against the lane table and bound to the lane's platform and the lock's interpreter (the C library from the libc fact before the ABI tag, `musl` included, `glibc` or `musl` read in any ASCII letter case at the start of the fact only and with no letter after it, with the error at that fact, and a libc fact or ABI tag that names another C library contradicting the lane whatever the status), version bounds (macOS Apple Silicon by macOS version, statically resolvable for 15 or newer and not supported for 14 or older; Linux x86-64 glibc checked for glibc 2.39 or newer, read from the libc fact in any ASCII letter case and not from the ABI tag, and CPython 3.12.13 or newer as a release; a version part with a leading zero or more than four digits, or more than one blank after `glibc`, is not read; a claim whose versions miss a bound or are not declared fails, and a real profile copies the status its versions meet or else records not-assessed, and on a lane without bounds copies the lane's status), the validator's lanes and version bounds kept equal to the table's, each bound naming a version the validator reads, a lane table that lists a lane twice (a usage error), evidence sources (with a comment line inside `jobs:`, job keys indented by four spaces, quoted, or followed by a tab or a comment, a quoted `jobs` key or one with a blank before its colon, and U+0085 inside a value, which name the job, and a deeper key, a longer name, no space after the colon, a comment right after a colon, an indented `jobs` key, a `jobs` block scalar, a `jobs` mapping in flow style, U+2028 inside a value, or a dot in the job name matched as another character, which name none, and a workflow file that is not UTF-8 or a path the system refuses, each reported as a finding that names why), the profile link (job, workflow, a workflow ref in another repository, including one whose name only starts with the record's, or with no workflow file or no ref, a ref holding `@`, which links, a record without its workflow ref or its repository, which links to no profile and is named by the field it lacks, and a real profile's GitHub Actions record) and the comparison of every declared fact (with the layer count, runtimes, backend API, runner, and `not-applicable` facts each checked on their own, and declared values in the result not counting), a blocked stage, fallback from an NPU and to another GPU, and CI evidence (each run and commit field, the workflow ref included, also when declared rather than observed or inferred, a run attempt of 0, and a run URL for another attempt or with an attempt longer than any `run_attempt`, while one with no attempt names the run). Command-line exit codes 0, 1, and 2, an unwritable or unusable schema, a schema given with `--schema` that accepts a document or profile the rules across fields cannot read (exit 2, not a traceback), and an exception in a rule under the default schema, however its path is written, which is raised, not reported as a usage error, an unwritable summary or one whose path loops, a summary that names the schema, the profile, the lane table, a document (the document also through a hard link or another spelling of its path), or a workflow file a real profile of the run names as its evidence source (also through a hard link, also when that file is missing, which it then does not create while it writes a summary elsewhere, whether that profile is given with `--profile` or as a document, and also when `synthetic` is a false value other than `false`), which leaves every input unchanged, a summary whose cells escape Markdown and that lists at most 200 findings, file names (including one that starts with `::`) and document text that cannot forge an output line, including the runner's `##[` command form anywhere in a line, a command-line error that repeats such an argument, a secret-shaped one, or such a program name, and a secret-shaped value or file name that the printed lines and the summary do not show, also after an escape the finding writes just before the value (`\n` from `repr`, `~1` in a JSON Pointer). | +| `scripts/ci/tests/test_environment_probe.py` | `unittest`, 72 tests of `scripts/ci/collect_environment.py` on fake machines built in temporary directories, with the interpreter facts fixed to match each machine: a GitHub-hosted runner VM, a Hyper-V VM named only by its VMBus, a machine with no virtualization signal (not reported as bare metal), Docker in a VM, a container that leaves none of the markers (told apart from the VM by its PID namespace), an Apptainer container in the initial PID namespace, signals that do and do not show a VM (EC2 metal instances and an EC2 Arm VM, WSL 1 with the hypervisor flag), rootless Podman with a non-root user and with root, WSL2 with current and older kernels and a distribution in its own PID namespace, an emulated process (recorded on the runtime, not the container layer), an old kernel without `/proc/sys/kernel/arch`, the interpreter's build architecture (from the ELF header only when its byte order and class match the name, so a big-endian ppc64, aarch64 or armeb is not read as ppc64le, aarch64 or armv7, nor a 32-bit x32, aarch64 ILP32, riscv32, loongarch32 or 31-bit s390 build as its 64-bit name) and free-threading flag, a platform tag overridden by `_PYTHON_HOST_PLATFORM` or malformed (recorded as `unknown`), the C library version read only through `os.confstr`, Node on Power named from Node's own byte order, not Python's, kernel and AppArmor text with control characters, an AppArmor `enabled` value other than `Y` or `N` and a newline (also padded, or longer than the 16 bytes read), the service manager, a Kubernetes pod, systemd in a container, container markers that disagree, Docker in a user namespace of its own, the kernel release read from `/proc` with `uname` only as an inferred fallback, os-release values that fail their checks, a FIFO in place of a file, and the provenance each derived fact keeps; the real and saved user IDs and a process that can regain root, cgroup version and delegation (not delegated without write access to the directory, `cgroup.procs`, `cgroup.threads` where it exists, or `cgroup.subtree_control`, with a `cgroup.threads` that cannot be looked up counted as present, and left unknown for real or saved root and for permitted capabilities, and for a cgroup path that leaves `/sys/fs/cgroup`), security modules, effective and permitted capabilities, missing files giving `unknown` values, and a failing probe kept apart from the other values; Windows, validated with the policy, which fails closed, the user IDs on FreeBSD and macOS and none under Cygwin, and the FreeBSD platform tag; pull request, `pull_request_target`, and push events, a run attempt up to the schema's largest integer, 4294967295 (a larger one is `unknown`), a manual dispatch from a branch with `#`, a comma, an apostrophe, `@`, a non-ASCII letter, or a trailing U+3000, checked out by its name as actions/checkout does, whose ref, workflow ref, and commit (from a loose ref or `packed-refs`) are recorded and which links to the profile, a workflow ref that is `unknown` for a branch with a space, `~`, `:`, `\`, a control or bidirectional formatting character, or a lone surrogate, or longer than 200 characters, a ref read in the same way, the characters a ref may hold checked against the validator's on every code point, malformed, unparsable, and missing event files and a missing event path, a checkout ref that leaves `.git`, that holds `..` or `@{`, a part that is empty, starts with `.`, or ends with `.lock` or `.`, a space, a control or bidirectional formatting character, or a byte that is not UTF-8 (not read as U+FFFD, the name of another branch), or that is longer than 200 characters, which is not followed even when `packed-refs` lists it, refs that hold `{`, `@`, or `.lock` elsewhere and are followed, `packed-refs` lines that end at `\n` only, a `packed-refs` line that is not UTF-8 (not read as U+FFFD, and not stopping the lookup), a ref that is not ASCII under a file system encoding other than UTF-8, which is not followed, and a `GITHUB_REF` or `GITHUB_WORKFLOW_REF` that is not ASCII under such an encoding, recorded as `unknown`, a loose ref that does not exist, looked up in `packed-refs`, and one that cannot be read or holds no commit ID (a symbolic ref), which is not, and the variable allowlist (which no longer holds the WSL variables); `node` run with an empty environment (on Windows, `SystemRoot` alone), looked for only on the `PATH` given (an empty `PATH` finds none, an unset one the default directories, and on Windows `node.exe` alone is looked for, not a `node.bat`, `node.cmd`, `node.com` or `node` earlier on `PATH`, without `shutil.which` and so in the same way on CPython 3.11 and 3.12, an empty entry names no directory, and the current directory is searched only when `PATH` names it, not when `PATH` is unset and the default directories list it first), failing, absent, skipped, reporting values that fail their checks, writing more than 4 KiB (and then ignoring the closed pipe), or not finishing in time (also after closing its output, by the deadline the read had), and a timer that fires after `node` was reaped, also after a failed read or right after the wait, which then sends no signal, or as the read ends, which holds back the reap until its signal is sent; stage option errors, including a value of every secret shape, which both the collector and the validator refuse; exit status 1 after a probe failure with the record still written; an unwritable output or summary; a summary that names the record's own file, also through a hard link or another spelling of its path, which leaves the record valid JSON; an output file name that would start a workflow command or is shaped like a token, which the printed line does not show, and a command-line error that repeats such an argument, a secret-shaped one, or such a program name, with the collector's line handling checked against the validator's; a record holding a secret-shaped value, which is not written, also when the JSON text writes the character before it as an escape such as `\n`, with the collector's secret shapes checked against the validator's; a summary with Markdown in a recorded value; and the tables both scripts keep (architecture names and which of them are 32-bit, platform tags, stages, the overall verdict, and the layer kind of every technology the collector writes), checked against each other and, for the platform tags, against expected values. One runs the probe's `node -p` expression with an empty environment on the `node` on `PATH`, when there is one, and checks that it gives every value the probe reads, the byte order included; a version-manager shim that needs `HOME` fails it. Two run the collector on the machine running the tests, with secret-shaped values in its environment, and check that no secret, host name, or home directory reaches the record and that the record is valid. Four read the collector's source and check its imports, that it never reads the whole environment or the host name, starts one subprocess without a shell, with an empty environment or, on Windows, `SystemRoot` alone, and discards its stderr, and opens files for reading except its outputs. | No test covers authentication, authorization, scope separation, approval, Tasks, retrieval, Browser sessions, realtime, i18n, or WebMCP, because none of those features exist. @@ -68,6 +70,18 @@ So an input change fails, until the locks are regenerated, when on the target it `scripts/ci/tests/test_check_test_lock.py` holds 49 regression tests for these rules, which use the standard library's `unittest`; run as a script, it exits 5 when it runs no test. CI runs the file directly with `python -I` rather than through `unittest discover`, which would put `scripts/ci/tests/` on the import path. 42 copy the two locks and two inputs into a temporary directory, change them, for example by dropping `psycopg`'s `binary` extra, adding `; sys_platform == "win32"` to `boto3`, adding `requests>=2.32 ; platform_release < "6"`, or replacing `Django` with a direct URL (a few change several files, and one leaves them as committed), run the check there with `python -I`, and expect it to fail with a given message or to pass; a test whose edit no longer changes its file fails. The other seven call the check's functions on made-up inputs, including with the running machine reporting itself as Windows, and on input files that are not UTF-8 or nest a marker 2,000 parentheses deep. +The compatibility model in [CI-MATRIX.md](CI-MATRIX.md) has its own scripts. They use the standard library only, so they need no installed package and no Django settings; CI runs them with CPython 3.12.15 in an environment with no packages: + +```bash +# From the repository root +python -I scripts/ci/collect_environment.py --profile-id linux-x64-gha --output "${TMPDIR:-/tmp}/linux-x64-gha.result.json" +python -I scripts/ci/validate_compatibility.py --require-real --dependency-strategy docs/DEPENDENCY-STRATEGY.md --profile tests/compatibility/profiles/linux-x64-gha.json "${TMPDIR:-/tmp}/linux-x64-gha.result.json" tests/compatibility/profiles/*.json +python -I scripts/ci/tests/test_compatibility_schema.py --verbose +python -I scripts/ci/tests/test_environment_probe.py --verbose +``` + +The collector exits 0 when it wrote the record and detection passed, 1 when a probe failed (the record is still written), and 2 on a usage error, an unwritable record or summary, a record that would hold a secret-shaped value, which it then does not write, or a `--summary` that names the record's own file, which it then leaves as written; the validator exits 0 when every document is valid, with or without warnings, 1 when any is invalid, and 2 on a usage error or an unusable file, schema, lane table, or summary, including a lane table that lists a lane twice and a `--summary` that names its schema, profile, lane table, a document, or a workflow file a real profile of the run names as its evidence source, also one that is missing. The profile describes a GitHub-hosted `ubuntu-24.04` runner, so a record collected elsewhere, for example outside GitHub Actions, in a container, on another distribution, or as root, fails against it with `profile-link`, `profile-mismatch`, `profile-unverified`, or a `policy-*` code; that is the expected result, not a defect. To check only that such a record is well formed, validate it without `--profile` and with `--no-policy`. A synthetic fixture is validated with its own profile, for example `--profile tests/compatibility/fixtures/windows-arm64-wsl2-podman.profile.json tests/compatibility/fixtures/windows-arm64-wsl2-podman.result.json`. Each test file exits 5 when it runs no test, and is run directly with `python -I`, as for the lock check's tests. + In Windows PowerShell the environment is activated with `.venv\Scripts\Activate.ps1` instead of `source .venv/bin/activate`, and the other commands are unchanged. Native Windows cannot produce a working environment from this Linux-resolved lock until a Windows lane exists: on Windows x64 the sync installs without `tzdata`, which Django and psycopg require on Windows, so `uv pip check` and time-zone handling fail; on Windows on Arm the sync itself fails because the pinned `autobahn`, `cryptography`, and `psycopg-binary` releases have no `win_arm64` wheels. Use WSL2 (Ubuntu) instead. The ASGI integration tests must exercise `config.asgi.application`, including `/core/health/`, `/agent/health/`, `/agent/openapi.json`, the removed `/api/health/` route, and wrong-slash `/agent/*` paths, which must return 404 without a `Location` header. @@ -117,7 +131,7 @@ The default browser is the binary paired with `@playwright/test`. If its downloa ## Continuous integration -Added 2026-10-10. `.github/workflows/ci.yml` runs the backend and frontend commands of "Current scaffold checks", plus `npx next typegen` and `npx tsc --noEmit`, on every pull request to `main`, every push to `main`, and on manual dispatch. It covers the checked lane only: GitHub-hosted `ubuntu-24.04` x86-64 runners, uv 0.12.24 with CPython 3.12.15 and the hash lock, and Node 24.21.0 (from `.nvmrc`) with npm 11.21.0. The first version of the workflow changed no test, lock, or package manifest; the pytest and Vitest jobs, added later on 2026-10-10, came with the backend test lock and the Vitest packages, and left the four original jobs unchanged. A workflow file is not evidence that anything ran or passed: cite the run, its commit SHA, the job, and the summary line from its log, as for any other check. A `pull_request` run tests the merge of the pull request's head into its base as they were when the run started (`refs/pull//merge`), not the head commit alone: record both the head SHA and the merge commit SHA that the checkout step logs, and treat the run as stale once the head or the base moves. +Added 2026-10-10. `.github/workflows/ci.yml` runs the backend and frontend commands of "Current scaffold checks", plus `npx next typegen` and `npx tsc --noEmit`, on every pull request to `main`, every push to `main`, and on manual dispatch. It covers the checked lane only: GitHub-hosted `ubuntu-24.04` x86-64 runners, uv 0.12.24 with CPython 3.12.15 and the hash lock, and Node 24.21.0 (from `.nvmrc`) with npm 11.21.0. The first version of the workflow changed no test, lock, or package manifest; the pytest and Vitest jobs, added later on 2026-10-10, came with the backend test lock and the Vitest packages, and left the four original jobs unchanged. The compatibility job, added 2026-10-11, installs no package and left the six earlier jobs unchanged. A workflow file is not evidence that anything ran or passed: cite the run, its commit SHA, the job, and the summary line from its log, as for any other check. A `pull_request` run tests the merge of the pull request's head into its base as they were when the run started (`refs/pull//merge`), not the head commit alone: record both the head SHA and the merge commit SHA that the checkout step logs, and treat the run as stale once the head or the base moves. | Job (check name) | What it runs | |---|---| @@ -127,11 +141,13 @@ Added 2026-10-10. `.github/workflows/ci.yml` runs the backend and frontend comma | `E2E / Playwright Chromium / dev Next` | The same with `PLAYWRIGHT_NEXT_SERVER=dev`. Only `next dev` can serve `/_next/mcp`, so this run, not the production one, is the check that `experimental.mcpServer: false` still holds. | | `Backend / pytest / CPython 3.12.15 / Linux x64` | The backend job's install with the integrated test lock instead of the backend lock, with `UV_NO_CONFIG` set so uv ignores configuration files in the checkout, and with `python-version` given to `astral-sh/setup-uv` so it does not run an interpreter from the checkout; runs `scripts/ci/check_test_lock.py` with `python -I` against the two locks, their requirement inputs, and the installed packages' metadata, then its regression tests by running `scripts/ci/tests/test_check_test_lock.py` with `python -I`, both before any other step runs Python from the checkout; records `python -VV` and the pytest version and fails on any other Python version; then, in `backend/`, `scripts/ci/check_test_parity.py` and `python -m pytest`. | | `Frontend / Vitest / Node 24 / Linux x64` | The frontend job's npm 11.21.0 install and `npm ci`, then `npm run test:unit`. | +| `Compatibility / Schema & Probe / Linux x64` | With `UV_NO_CONFIG` set and `python-version` given to `astral-sh/setup-uv`, as in the pytest job, and Node from `.nvmrc`: creates a CPython 3.12.15 environment with `uv venv --managed-python` and installs nothing into it; runs `scripts/ci/collect_environment.py` with `python -I` to record the runner as a result for the `linux-x64-gha` profile, writing a summary table to the job summary; validates that record and every committed profile with `scripts/ci/validate_compatibility.py --require-real --dependency-strategy docs/DEPENDENCY-STRATEGY.md --profile tests/compatibility/profiles/linux-x64-gha.json`, also after a probe failure, which leaves a partial record; then, even when an earlier step failed, runs `scripts/ci/tests/test_compatibility_schema.py` and `scripts/ci/tests/test_environment_probe.py` with `python -I`, as two steps. Only `detect` runs, so the record's `overall` is `incomplete`; [CI-MATRIX.md](CI-MATRIX.md) "What a passing job means" states what a pass shows. | - Both E2E jobs upload `frontend/test-results/`, which holds the saved screenshots, as a run artifact kept for 7 days, whether the tests passed or failed. Inspecting the screenshots stays a reviewer's step, as above. -- No job depends on another, so a failure in one never turns another into a skipped check. When the four original jobs first passed on `main`, their check names became eligible as required status checks; the pytest and Vitest checks become eligible once they have passed. Making a check required is a GitHub repository setting, not a file in this repository, and changing it needs admin access to the repository. +- The compatibility job uploads its record, `linux-x64-gha.result.json`, as the artifact `compatibility-linux-x64-attempt-`, kept for 90 days, whether or not validation passed, unless the run was cancelled. The collector does not write a record that holds a secret-shaped value. The record holds the runner's OS, kernel, CPU, runtime, and isolation facts, the run's commit IDs, and no environment variable outside the collector's allowlist. +- No job depends on another, so a failure in one never turns another into a skipped check. When the four original jobs first passed on `main`, their check names became eligible as required status checks; the pytest, Vitest, and compatibility checks become eligible once they have passed. No change in this repository makes a check required. Making a check required is a GitHub repository setting, not a file in this repository, and changing it needs admin access to the repository. - A newer run for the same pull request cancels the older one. Every push to `main` and every manual dispatch runs in its own concurrency group, so none of those runs is cancelled or dropped from the queue. -- Not covered by CI: other operating systems, CPU architectures, and accelerators; Python 3.15 and 3.15t; the manual route checks against `runserver` and Uvicorn above, for which no API collection exists (see "API collection testing" below); the Python Playwright browser install; accessibility and pixel-diff checks; secret scanning; and dependency alerts. +- Not covered by CI: other operating systems, CPU architectures, and accelerators, of which only synthetic fixtures exist; the stages after `detect` on any lane; Python 3.15 and 3.15t; the manual route checks against `runserver` and Uvicorn above, for which no API collection exists (see "API collection testing" below); the Python Playwright browser install; accessibility and pixel-diff checks; secret scanning; and dependency alerts. - A green run shows that these commands passed on that commit and runner. It is not a review and not an approval ([CODE-REVIEW.md](CODE-REVIEW.md) "Independence and authority"). A change to any workflow file goes through the security review ([CODE-REVIEW.md](CODE-REVIEW.md) step 9). ## API collection testing diff --git a/scripts/ci/collect_environment.py b/scripts/ci/collect_environment.py new file mode 100644 index 0000000..c99a00e --- /dev/null +++ b/scripts/ci/collect_environment.py @@ -0,0 +1,1449 @@ +"""Record the environment this process runs in as a compatibility result (docs/CI-MATRIX.md). + +The collector reads the following, and nothing else: + +- /proc, /sys, /etc/os-release or /usr/lib/os-release, /run/.containerenv + and /run/systemd/container, and whether /.dockerenv, /.singularity.d and + /run/systemd/system exist. +- The first 20 bytes of the interpreter's executable (its ELF header), when the + build architecture is not otherwise known. +- The environment variables in ENV_ALLOWLIST. +- The pull request head and base in the GitHub Actions event file. +- The checkout's .git/HEAD, and the loose ref or packed-refs line it points to. +- The first 4 KiB that `node -p` writes, run with an empty environment (on + Windows, SystemRoot only) and stopped after 30 seconds. node is looked for in + the PATH directories only, so on Windows not in the current directory unless + PATH names it, also when PATH is unset, and there only as node.exe, whatever + PATHEXT lists. + +Files are read only when they are regular files, and are opened so that a FIFO +cannot block the probe. + +It uses no network, never escalates privileges, and writes only the output file +and the optional summary. It records no host name, account name, home +directory, network address or other environment variable. It refuses --stage +values, and does not write a record, that look like a token or a private key. + +On Linux it fills in the layers, identity and isolation facts. On other systems +it records the OS family, the user and group IDs on FreeBSD (os.getresuid) +and macOS (os.geteuid, os.getuid), the Python and Node runtimes and the run, +and leaves the layer kind and the isolation facts unknown, so the +least-privilege policy fails there until a probe measures them. + +Each value it records is marked in one of these ways: + +- observed: read on this machine. The evidence names the file, variable or call. +- inferred: derived from evidence that is named. +- declared: stated, not measured. The collector declares only the detect + stage's escalation (none: this script never changes its privileges) and the + escalation a --stage-escalation option gives. +- unknown or not-applicable: with the reason. + +An absent or unreadable file gives an unknown value. It is not an error. An +unexpected exception in one probe is recorded in probe_errors and fails the +detect stage. The other probes' values stay in place, so a record can show +which part of detection failed. + +Standard library only. Usage, from the repository root: + python -I scripts/ci/collect_environment.py --output result.json + [--profile-id ID] [--summary FILE] [--no-node] + [--stage NAME=STATUS] [--stage-reason NAME=TEXT] + [--stage-evidence NAME=REF] [--stage-escalation NAME=VALUE] + +The --stage options record what other steps of the job did in the stages after +detect. Every stage they do not name is not-run. + +Exit status: +- 0: the record was written and detection passed. +- 1: the record was written but a probe failed. +- 2: a usage error, or the record or the summary could not be written. The + record is written first, and the message says which one failed. A record + that looks like it holds a secret is not written. +""" + +import argparse +import datetime +import json +import os +import pathlib +import platform +import re +import shutil +import signal +import stat +import struct +import subprocess +import sys +import sysconfig +import threading +import time + +COLLECTOR = {"name": "collect-environment", "version": 1} +ROOT = pathlib.Path(__file__).resolve().parents[2] +TOOL = "tool:collect-environment" + +# The only environment variables the collector reads. Values of PRESENCE_ONLY +# variables are never recorded: PATH is used only to find node, +# GITHUB_EVENT_PATH only to open the event file, and SystemRoot only to start +# node on Windows. +ENV_ALLOWLIST = frozenset({ + "GITHUB_ACTIONS", "GITHUB_EVENT_NAME", "GITHUB_EVENT_PATH", "GITHUB_SHA", "GITHUB_REF", + "GITHUB_REPOSITORY", "GITHUB_SERVER_URL", "GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", + "GITHUB_WORKFLOW_REF", "GITHUB_JOB", "RUNNER_ENVIRONMENT", "RUNNER_OS", "RUNNER_ARCH", + "ImageOS", "ImageVersion", "container", "KUBERNETES_SERVICE_HOST", "PATH", + "_PYTHON_HOST_PLATFORM", "_PYTHON_SYSCONFIGDATA_NAME", "_PYTHON_SYSCONFIGDATA_PATH", + "APPTAINER_CONTAINER", "SINGULARITY_CONTAINER", "SystemRoot", +}) +# sysconfig reads these even under python -I; when one is set, its values are not recorded. +SYSCONFIG_OVERRIDES = ("_PYTHON_HOST_PLATFORM", "_PYTHON_SYSCONFIGDATA_NAME", "_PYTHON_SYSCONFIGDATA_PATH") +PRESENCE_ONLY = frozenset({"KUBERNETES_SERVICE_HOST", "PATH", "GITHUB_EVENT_PATH", "APPTAINER_CONTAINER", "SINGULARITY_CONTAINER", "SystemRoot", *SYSCONFIG_OVERRIDES}) + +STAGES = ("detect", "install", "initialize", "execute", "correctness", "security", "performance") +STATUSES = ("passed", "failed", "blocked", "not-run", "not-applicable") +ESCALATIONS = ("none", "sudo", "root-login", "setuid", "other") +SLUG = re.compile(r"[a-z0-9][a-z0-9._+-]{0,63}") +# sysconfig.get_platform(), which on FreeBSD carries the release in capitals, such as freebsd-14.1-RELEASE-amd64. +PLATFORM_TAG = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+-]{0,63}") +SHA = re.compile(r"[0-9a-f]{40}") +EVIDENCE = re.compile(r"(file|env|python|cmd|event|git|tool|doc|profile|workflow|log|artifact):[!-~][ -~]{0,299}") +PRINTABLE = re.compile(r"[ -~]{1,200}") + +ARCH_ALIASES = { + "amd64": "x86_64", "x64": "x86_64", "arm64": "aarch64", "i386": "x86", "i486": "x86", + "i586": "x86", "i686": "x86", "ia32": "x86", "armv7l": "armv7", "armv8l": "armv7", "arm": "armv7", + "loong64": "loongarch64", "powerpc64le": "ppc64le", +} +# (process, kernel) architecture pairs that run without emulation: 32-bit x86 code on an x86-64 kernel. +# 32-bit Arm on aarch64 is not listed: many recent cores have no AArch32, so the pair is recorded as unknown. +COMPATIBLE = frozenset({("x86", "x86_64")}) +# ELF e_machine values (elf.h) for the architectures in docs/CI-MATRIX.md. +ELF_MACHINES = {3: "x86", 40: "armv7", 62: "x86_64", 183: "aarch64", 243: "riscv64", 22: "s390x", 258: "loongarch64", 21: "ppc64le"} +# Each name above is for one byte order and one ELF class. s390x is big-endian and the others +# little-endian (ppc64, aarch64_be and armeb share their e_machine with ppc64le, aarch64 and armv7); +# x86 and armv7 are 32-bit and the others 64-bit (riscv32, 31-bit s390, loongarch32, x32 and +# aarch64 ILP32 share theirs with a 64-bit name). +ELF_BIG_ENDIAN = frozenset({"s390x"}) +ELF_32_BIT = frozenset({"x86", "armv7"}) +KNOWN_ARCHES = frozenset(ELF_MACHINES.values()) +OS_FAMILIES = {"Linux": "linux", "Windows": "windows", "Darwin": "macos", "FreeBSD": "freebsd"} +# Capability numbers from linux/capability.h. +CAPABILITIES = ( + "chown", "dac_override", "dac_read_search", "fowner", "fsetid", "kill", "setgid", "setuid", + "setpcap", "linux_immutable", "net_bind_service", "net_broadcast", "net_admin", "net_raw", + "ipc_lock", "ipc_owner", "sys_module", "sys_rawio", "sys_chroot", "sys_ptrace", "sys_pacct", + "sys_admin", "sys_boot", "sys_nice", "sys_resource", "sys_time", "sys_tty_config", "mknod", + "lease", "audit_write", "audit_control", "setfcap", "mac_override", "mac_admin", "syslog", + "wake_alarm", "block_suspend", "audit_read", "perfmon", "bpf", "checkpoint_restore", +) +SECCOMP = {"0": "disabled", "1": "strict", "2": "filter"} +# The fixed inode of the initial user namespace (PROC_USER_INIT_INO in linux/proc_ns.h). +USER_NS_INIT_INO = 0xEFFFFFFD +# The fixed inode of the initial PID namespace (PROC_PID_INIT_INO). +PID_NS_INIT_INO = 0xEFFFFFFC +# The same characters as CONTROL in validate_compatibility.py; a test keeps plain() equal to the validator's. +CONTROL = re.compile("[\x00-\x1f\x7f-\x9f\u061c\u200e\u200f\u2028-\u202e\u2066-\u2069\ud800-\udfff]") +# A character git allows in a ref: none of space, "~", "^", ":", "?", "*", "[" or "\", and none of the characters +# CONTROL refuses in any value, taken from CONTROL itself. +REF_CHAR = "[^ ~^:?*\\[\\\\" + CONTROL.pattern[1:] +# GITHUB_REF, and the ref .git/HEAD names: actions/checkout checks a branch out by its name, so a run from a branch +# named with any of these characters records its ref, if it is ASCII or the file system encoding is UTF-8, and its +# commit unless checkout_sha does not follow that name. Within the schema's 200 characters. +GIT_REF = rf"(?=.{{1,200}}\Z)refs/{REF_CHAR}+" +# GITHUB_WORKFLOW_REF is //@, and the validator links a run to its profile by it, so it +# is recorded for any of those refs. The whole value stays within the schema's 200 characters. +WORKFLOW_REF = rf"(?=.{{1,200}}\Z)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]{{1,150}}@refs/{REF_CHAR}+" +# The same shapes as SECRET_SHAPES in validate_compatibility.py; a test keeps them equal. None starts with \b, so +# a shape is still found in the JSON text after an escape such as \n or \u200e. +SECRET_SHAPES = ( + ("GitHub token", re.compile(r"gh[opsur]_[A-Za-z0-9]{30,}")), + ("GitHub token", re.compile(r"github_pat_[A-Za-z0-9_]{20,}")), + ("AWS access key ID", re.compile(r"(AKIA|ASIA)[0-9A-Z]{16}(?![0-9A-Z])")), + ("private key", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")), + ("Slack token", re.compile(r"(?:xox[abposr]|xapp)-[A-Za-z0-9-]{10,}")), + # A JSON Web Token: "eyJ", at least 8 more base64url characters and ".", twice. The run that holds the first + # "eyJ" is matched from its start, so each run is tested once and a long run does not make the search quadratic. + ("JSON Web Token", re.compile(r"(?= 5 and tail: + mounts[fields[4]] = tail[0] + return mounts, None + + def namespace_inode(self, name): + return os.stat(self.path(f"/proc/self/ns/{name}")).st_ino + + # -- sections ---------------------------------------------------------- + + def layers(self): + if self.system != "Linux": + family = OS_FAMILIES.get(self.system) + reason = f"this probe reads Linux interfaces only; this system is {self.system or 'unidentified'}" + return [{ + "kind": unknown(reason), + "technology": unknown(reason), + "os": { + "family": observed(family, "python:platform.system") if family else unknown(reason), + "name": unknown(reason), + "version": unknown(reason), + "kernel": unknown(reason), + }, + "cpu": {"arch": unknown(reason), "execution": unknown(reason)}, + "service_manager": {"name": unknown(reason), "scope": unknown(reason)}, + }] + + _, hypervisor = self.cpuinfo() + kernel_arch = self.kernel_arch() + kernel_fact = self.kernel_release() + + container_names, container_evidence, containerenv = self.container_signals() + in_container = bool(container_evidence) + wsl = self.wsl() + dmi_technology, dmi_evidence = self.dmi_technology() + if dmi_technology == "amazon" and not hypervisor: + # EC2 metal instances name Amazon EC2 in DMI but run no hypervisor; their product name is + # .metal or .metal-. Arm /proc/cpuinfo has no hypervisor flag to tell. + product, _ = self.read("/sys/class/dmi/id/product_name", 256) + if product is not None and re.search(r"\.metal(-[0-9a-z]+)?$", product.strip()): + dmi_technology, dmi_evidence = None, None + try: + initial_pid_namespace = self.namespace_inode("pid") == PID_NS_INIT_INO + except OSError: + initial_pid_namespace = None + + # The layer that runs the kernel. + kernel_layer = {"os": {"family": observed("linux", "python:platform.system"), "kernel": kernel_fact}} + if wsl == "wsl1": + reason = "WSL 1 runs Linux programs on Windows without a VM" + kernel_layer["kind"] = unknown(reason) + kernel_layer["technology"] = unknown(reason) + kernel_layer["os"]["kernel"] = unknown("WSL 1 runs no Linux kernel; Windows reports this release string") + elif wsl == "microsoft": + reason = "a Microsoft kernel that is not a known WSL release" + kernel_layer["kind"] = unknown(reason) + kernel_layer["technology"] = unknown(reason) + elif wsl == "wsl2": + kernel_layer["kind"] = inferred("wsl2", "file:/proc/sys/kernel/osrelease") + kernel_layer["technology"] = inferred("wsl2", "file:/proc/sys/kernel/osrelease") + elif hypervisor or dmi_technology: + vm_evidence = [e for e in (("file:/proc/cpuinfo#flags:hypervisor" if hypervisor else None), dmi_evidence) if e] + kernel_layer["kind"] = inferred("vm", *vm_evidence) + kernel_layer["technology"] = inferred(dmi_technology, dmi_evidence) if dmi_technology else unknown("a hypervisor is present but neither DMI nor a VMBus names it") + else: + reason = "no VM or WSL signal was found; that does not show bare metal" + kernel_layer["kind"] = unknown(reason) + kernel_layer["technology"] = unknown(reason) + + hidden = "not visible from inside the container" + if in_container: + kernel_layer["os"]["name"] = unknown(f"the distribution of the system that runs the container is {hidden}") + kernel_layer["os"]["version"] = unknown(f"the version of the system that runs the container is {hidden}") + kernel_layer["service_manager"] = {"name": unknown(f"the service manager outside the container is {hidden}"), "scope": unknown(hidden)} + elif initial_pid_namespace is not True and wsl != "wsl2": + # A container that leaves none of the signals above still has its own PID namespace, + # and its os-release and PID 1 are not the kernel layer's. + why = "this process is not in the initial PID namespace, or that cannot be read, so os-release and PID 1 may belong to a container or sandbox" + kernel_layer["os"]["name"] = unknown(why) + kernel_layer["os"]["version"] = unknown(why) + kernel_layer["service_manager"] = {"name": unknown(why), "scope": unknown(why)} + else: + # Taken to be the kernel layer's because no container signal was found and the PID + # namespace is the initial one, or under WSL 2 the distribution's own, which WSL 2 gives + # every distribution. A chroot, or a container that shares that namespace and leaves no + # signal, would be read here too, so these facts are inferred, not observed. + pid_evidence = "file:/proc/self/ns/pid#inode" if initial_pid_namespace else "file:/proc/sys/kernel/osrelease" + kernel_layer["os"].update({key: as_inferred(fact, pid_evidence) for key, fact in self.distribution().items()}) + kernel_layer["service_manager"] = {key: as_inferred(fact, pid_evidence) for key, fact in self.service_manager().items()} + if kernel_arch is not None: + layer_arch = observed(kernel_arch, "file:/proc/sys/kernel/arch") + else: + # uname and /proc/cpuinfo both follow personality changes and qemu-user, so neither names the kernel's architecture. + layer_arch = unknown("without /proc/sys/kernel/arch, uname cannot be told apart from an emulated or personality-changed machine name") + kernel_layer["cpu"] = { + "arch": layer_arch, + "execution": unknown("whether the layer below emulates this CPU is not visible from inside it"), + } + kernel_layer = order_layer(kernel_layer) + + layers = [] + if known_kind(kernel_layer) in ("vm", "wsl2"): + host_reason = "a guest cannot see the physical host" + host_family = inferred("windows", "file:/proc/sys/kernel/osrelease") if wsl == "wsl2" else unknown(host_reason) + layers.append(order_layer({ + "kind": inferred("host", *kernel_layer["kind"]["evidence"]), + "technology": not_applicable("the outermost host runs on hardware"), + "os": {"family": host_family, "name": unknown(host_reason), "version": unknown(host_reason), "kernel": unknown(host_reason)}, + "cpu": {"arch": unknown(host_reason), "execution": unknown(host_reason)}, + "service_manager": {"name": unknown(host_reason), "scope": unknown(host_reason)}, + })) + layers.append(kernel_layer) + + if in_container: + container = {"kind": inferred("container", *container_evidence)} + if len(container_names) == 1: + container["technology"] = inferred(next(iter(container_names)), *container_evidence) + elif container_names: + container["technology"] = unknown(f"the container signals name different engines ({', '.join(sorted(container_names))})", *container_evidence) + else: + container["technology"] = unknown("the container signals do not name the engine", *container_evidence) + container["os"] = dict(self.distribution(), family=observed("linux", "python:platform.system"), kernel=not_applicable("a container shares the kernel of the layer below")) + # The container runs on the kernel layer's CPU, and its programs need not all be built for + # one architecture, so each runtime records its own architecture and whether it is emulated. + per_runtime = "a container's programs can be built for different architectures; each runtime records its own" + container["cpu"] = {"arch": unknown(per_runtime), "execution": unknown(per_runtime)} + container["service_manager"] = self.service_manager() + rootless = containerenv.get("rootless") + if container_names != {"podman"}: + engine_mode = unknown("only Podman tells a container whether it runs rootless, and the markers do not name Podman alone") + elif rootless in ("0", "1"): + # The file is inside the container; Podman writes it, but an image could also hold one. + engine_mode = inferred("rootless" if rootless == "1" else "rootful", "file:/run/.containerenv#rootless") + else: + engine_mode = unknown("/run/.containerenv does not say whether the engine runs rootless") + orchestration_name = inferred("kubernetes", "env:KUBERNETES_SERVICE_HOST") if self.env("KUBERNETES_SERVICE_HOST") is not None else unknown("no orchestrator signal; that does not show there is none") + container["container"] = { + "engine_mode": engine_mode, + "privileged": unknown("no reliable signal from inside the container"), + "devices": unknown("the devices an engine passes in are not recorded inside the container"), + "orchestration": { + "name": orchestration_name, + "version": unknown("not visible from inside the container"), + "scope": unknown("not visible from inside the container"), + }, + } + layers.append(order_layer(container)) + return layers + + def kernel_release(self): + """The release the kernel reports in /proc; uname, which personality changes and qemu-user can rewrite, only as a fallback.""" + text, _ = self.read("/proc/sys/kernel/osrelease", 256) + if text is not None: + release = text.strip() + if PRINTABLE.fullmatch(release): + return observed(release, "file:/proc/sys/kernel/osrelease") + return unknown("the kernel release is empty or has unexpected characters", "file:/proc/sys/kernel/osrelease") + release = platform.release() + if PRINTABLE.fullmatch(release or ""): + return inferred(release, "python:platform.release") + return unknown("the kernel release is empty or has unexpected characters") + + def distribution(self): + values, evidence = self.os_release() + if values is None: + return {"name": unknown("no os-release file"), "version": unknown("no os-release file")} + name = values.get("ID", "").lower() + version = values.get("VERSION_ID", "") + return { + "name": observed(name, f"{evidence}#ID") if SLUG.fullmatch(name) else unknown("os-release has no usable ID", evidence), + "version": observed(version, f"{evidence}#VERSION_ID") if PRINTABLE.fullmatch(version) else unknown("os-release has no usable VERSION_ID", evidence), + } + + def service_manager(self): + if self.exists("/run/systemd/system"): + pid1, _ = self.read("/proc/1/comm", 64) + scope = observed("system", "file:/proc/1/comm") if pid1 is not None and pid1.strip() == "systemd" else unknown("PID 1 of this namespace is not systemd or cannot be read") + return {"name": observed("systemd", "file:/run/systemd/system"), "scope": scope} + reason = "systemd is not running as the service manager here; other managers are not probed" + return {"name": unknown(reason, "file:/run/systemd/system"), "scope": unknown(reason)} + + def identity(self): + # The real and saved user IDs matter too: a process whose real or saved ID is 0 can set its effective ID back to 0. + if self.system == "Linux": + fields, reason = self.status_fields() + if "Uid" not in fields or "Gid" not in fields: + return identity_fallback(reason or "/proc/self/status has no Uid or Gid line") + # Uid: real, effective, saved set-user-ID, file system. + real_uid, uid, saved_uid = (int(part) for part in fields["Uid"].split()[:3]) + gid = int(fields["Gid"].split()[1]) + source = "file:/proc/self/status#Uid" + ids = {"uid": observed(uid, source), "real_uid": observed(real_uid, source), "saved_uid": observed(saved_uid, source)} + gid_fact = observed(gid, "file:/proc/self/status#Gid") + elif self.system == "Windows": + # Whether a Windows process holds an elevated token is not probed, so superuser stays unknown. + identity = identity_fallback("this probe reads POSIX user IDs only") + identity["real_uid"] = not_applicable("Windows has no POSIX user IDs") + identity["saved_uid"] = not_applicable("Windows has no POSIX user IDs") + return identity + elif self.system == "FreeBSD" and hasattr(os, "getresuid"): + real_uid, uid, saved_uid = os.getresuid() + source = "python:os.getresuid" + ids = {"uid": observed(uid, source), "real_uid": observed(real_uid, source), "saved_uid": observed(saved_uid, source)} + gid_fact = observed(os.getegid(), "python:os.getegid") + elif self.system == "Darwin" and hasattr(os, "geteuid"): + # macOS has no getresuid. execve(2) saves the effective user ID, and this script never changes it. + uid = os.geteuid() + source = "python:os.geteuid" + ids = {"uid": observed(uid, source), "real_uid": observed(os.getuid(), "python:os.getuid"), "saved_uid": inferred(uid, source)} + gid_fact = observed(os.getegid(), "python:os.getegid") + else: + # Elsewhere, such as Cygwin or MSYS2 on Windows, user ID 0 does not name the administrator. + return identity_fallback("this probe reads user IDs on Linux, FreeBSD, and macOS only") + identity = {"superuser": observed(uid == 0, source), **ids, "gid": gid_fact} + if self.system != "Linux": + reason = "user namespaces are a Linux interface" + identity["user_namespace"] = {"initial": unknown(reason), "uid_map": unknown(reason)} + return identity + text, reason = self.read("/proc/self/uid_map") + if text is None: + uid_map = unknown(reason) + else: + rows = [[int(part) for part in line.split()] for line in text.splitlines() if line.strip()] + if not rows or any(len(row) != 3 or not all(0 <= part <= 4294967295 for part in row) for row in rows): + raise ValueError("unexpected /proc/self/uid_map format") + uid_map = observed(rows, "file:/proc/self/uid_map") + # A full identity map does not show the initial namespace: a nested one can have it too. + try: + initial = observed(self.namespace_inode("user") == USER_NS_INIT_INO, "file:/proc/self/ns/user#inode") + except OSError as exc: + initial = unknown(f"/proc/self/ns/user cannot be read ({type(exc).__name__})") + identity["user_namespace"] = {"initial": initial, "uid_map": uid_map} + return identity + + def isolation(self): + if self.system != "Linux": + return isolation_fallback("this probe reads Linux interfaces only") + fields, reason = self.status_fields() + status = "file:/proc/self/status" + + def field(name, convert): + if name not in fields: + return unknown(reason or f"/proc/self/status has no {name} line") + return observed(convert(fields[name]), f"{status}#{name}") + + out = { + "no_new_privs": field("NoNewPrivs", lambda v: {"0": False, "1": True}[v]), + "capabilities": {"effective": field("CapEff", decode_capabilities), "permitted": field("CapPrm", decode_capabilities), "bounding": field("CapBnd", decode_capabilities)}, + "seccomp": field("Seccomp", lambda v: SECCOMP[v]), + "apparmor": self.apparmor(), + "selinux": self.selinux(), + "cgroup": self.cgroup(fields), + } + # Containers get their own PID namespace; the initial one shows the process is not in one. + try: + initial = observed(self.namespace_inode("pid") == PID_NS_INIT_INO, "file:/proc/self/ns/pid#inode") + except OSError as exc: + initial = unknown(f"/proc/self/ns/pid cannot be read ({type(exc).__name__})") + out["pid_namespace"] = {"initial": initial} + return out + + def apparmor(self): + if not self.exists("/sys/module"): + reason = "/sys/module is not available" + return {"enabled": unknown(reason), "profile": unknown(reason)} + text, reason = self.read("/sys/module/apparmor/parameters/enabled", 16) + if text is None: + if self.missing("/sys/module/apparmor/parameters/enabled"): + return {"enabled": inferred(False, "file:/sys/module/apparmor/parameters/enabled#absent"), "profile": not_applicable("AppArmor is not in this kernel")} + return {"enabled": unknown(reason), "profile": unknown(reason)} + # The kernel writes Y or N and a newline; anything else, such as a file that was replaced or a longer + # value cut at 16 bytes, says nothing. + if text not in ("Y\n", "N\n"): + reason = "/sys/module/apparmor/parameters/enabled is not Y or N and a newline, as the kernel writes it" + return {"enabled": unknown(reason), "profile": unknown(reason)} + if text == "N\n": + return {"enabled": observed(False, "file:/sys/module/apparmor/parameters/enabled"), "profile": not_applicable("AppArmor is not enabled")} + # /proc/self/attr/current belongs to whichever major LSM comes first, so only the + # AppArmor-specific file (Linux 5.8 and later) is read. + label, _ = self.read("/proc/self/attr/apparmor/current", 512) + if label is not None and PRINTABLE.fullmatch(label.strip("\n\x00 ")): + return {"enabled": observed(True, "file:/sys/module/apparmor/parameters/enabled"), "profile": observed(label.strip("\n\x00 "), "file:/proc/self/attr/apparmor/current")} + return {"enabled": observed(True, "file:/sys/module/apparmor/parameters/enabled"), "profile": unknown("the AppArmor label of this process cannot be read")} + + def selinux(self): + lsms = self.lsm_list() + text, _ = self.read("/sys/fs/selinux/enforce", 16) + if text is not None and text.strip() in ("0", "1"): + mode = observed("enforcing" if text.strip() == "1" else "permissive", "file:/sys/fs/selinux/enforce") + # /proc/self/attr/current is SELinux's only when no other major LSM is listed before it. + if lsms is not None and "selinux" in lsms and not {"apparmor", "smack", "tomoyo"} & set(lsms[:lsms.index("selinux")]): + label, _ = self.read("/proc/self/attr/current", 512) + if label is not None and PRINTABLE.fullmatch(label.strip("\n\x00 ")): + return {"mode": mode, "context": observed(label.strip("\n\x00 "), "file:/proc/self/attr/current", "file:/sys/kernel/security/lsm")} + return {"mode": mode, "context": unknown("the SELinux context of this process cannot be read or attributed")} + if lsms is not None and "selinux" not in lsms: + return {"mode": inferred("disabled", "file:/sys/kernel/security/lsm"), "context": not_applicable("SELinux is not an active security module")} + if lsms is not None: + why = "SELinux is an active security module but selinuxfs is not mounted here" + return {"mode": unknown(why), "context": unknown(why)} + filesystems, reason = self.read("/proc/filesystems", 16384) + if filesystems is not None and not re.search(r"\bselinuxfs\b", filesystems): + return {"mode": inferred("disabled", "file:/proc/filesystems#selinuxfs-absent"), "context": not_applicable("SELinux is not in this kernel")} + why = "the kernel supports SELinux but neither securityfs nor selinuxfs is mounted here" if filesystems is not None else reason + return {"mode": unknown(why), "context": unknown(why)} + + def cgroup(self, status): + """The cgroup version from the file system mounted at /sys/fs/cgroup, and delegation for v2.""" + mounts, reason = self.mount_types() + if mounts is None: + return {"version": unknown(reason), "delegated": unknown(reason)} + evidence = "file:/proc/self/mountinfo#/sys/fs/cgroup" + top = mounts.get("/sys/fs/cgroup") + if top == "cgroup2": + version = observed("v2", evidence) + elif top == "tmpfs" and any(fstype == "cgroup" for point, fstype in mounts.items() if point.startswith("/sys/fs/cgroup/")): + hybrid = any(fstype == "cgroup2" for point, fstype in mounts.items() if point.startswith("/sys/fs/cgroup/")) + version = observed("hybrid" if hybrid else "v1", evidence) + else: + why = "nothing is mounted at /sys/fs/cgroup" if top is None else "/sys/fs/cgroup has an unexpected file system" + return {"version": unknown(why, evidence), "delegated": unknown(why)} + if version["value"] != "v2": + return {"version": version, "delegated": unknown("delegation is read for cgroup v2 only")} + # access(2) checks the real user ID, so a real, effective or saved ID of 0, or any capability, leaves delegation unknown. + uid = status.get("Uid", "").split() + if len(uid) < 4 or "0" in uid or any(status.get(name, "0").strip("0") for name in ("CapEff", "CapPrm")): + return {"version": version, "delegated": unknown("write access says nothing about delegation for a process with root or capabilities")} + text, reason = self.read("/proc/self/cgroup", 16384) + unified = [line[3:] for line in (text or "").splitlines() if line.startswith("0::")] + if not unified: + return {"version": version, "delegated": unknown(reason or "/proc/self/cgroup has no cgroup v2 line")} + relative = unified[0].lstrip("/") + directory = self.path("/sys/fs/cgroup") / relative + if ".." in relative.split("/") or not directory.is_dir(): + return {"version": version, "delegated": unknown("this process's cgroup directory is not visible here")} + # access(2) reports permission bits only; nothing is written to find out. The kernel's delegation model grants write + # access to the directory and its cgroup.procs, cgroup.threads (Linux 4.14 and later) and cgroup.subtree_control; + # without the last a process cannot enable controllers for its children, so that is not delegation. + names = ["cgroup.procs", "cgroup.subtree_control"] + if not self.missing(f"/sys/fs/cgroup/{relative}/cgroup.threads"): + names.append("cgroup.threads") + delegated = os.access(directory, os.W_OK) and all(os.access(directory / name, os.W_OK) for name in names) + evidence = ["file:/sys/fs/cgroup/#writable"] + [f"file:/sys/fs/cgroup//{name}#writable" for name in names] + return {"version": version, "delegated": observed(delegated, *evidence)} + + def sysconfig_overrides(self): + return [name for name in SYSCONFIG_OVERRIDES if self.env(name) is not None] + + def build_arch(self): + """The architecture the interpreter was built for, with its evidence. + + On Linux sysconfig.get_platform() follows uname, which emulation and + personality changes rewrite, so the build triplet is read instead. + """ + multiarch = getattr(sys.implementation, "_multiarch", None) + if isinstance(multiarch, str) and "-" in multiarch: + return normalize_arch(multiarch.split("-", 1)[0]), "python:sys.implementation._multiarch" + soabi = None if self.sysconfig_overrides() else sysconfig.get_config_var("SOABI") + match = re.fullmatch(r"[a-z]+-[0-9]+[a-z]*-([a-z0-9_]+)-linux-[a-z0-9]+", soabi or "") + if match: + return normalize_arch(match.group(1)), "python:sysconfig.SOABI" + try: + with open(os.path.realpath(sys.executable or "/nonexistent"), "rb") as handle: + header = handle.read(20) + except OSError: + return None, None + if len(header) == 20 and header[:4] == b"\x7fELF" and header[5] in (1, 2): + machine = int.from_bytes(header[18:20], "little" if header[5] == 1 else "big") + name = ELF_MACHINES.get(machine) + if name and header[4] == (1 if name in ELF_32_BIT else 2) and header[5] == (2 if name in ELF_BIG_ENDIAN else 1): + return name, "file:sys.executable#e_machine" + return None, None + + def python(self, reference): + version = platform.python_version() + overrides = self.sysconfig_overrides() + if overrides: + why = f"{', '.join(overrides)} is set, which changes what sysconfig reports" + plat_fact, soabi, raw_gil_disabled = unknown(why), None, None + else: + plat = sysconfig.get_platform() + plat_fact = observed(plat, "python:sysconfig.get_platform") if isinstance(plat, str) and PLATFORM_TAG.fullmatch(plat) else unknown("sysconfig.get_platform() gave an unexpected value") + soabi = sysconfig.get_config_var("SOABI") + raw_gil_disabled = sysconfig.get_config_var("Py_GIL_DISABLED") + if self.system == "Linux": + arch, arch_evidence = self.build_arch() + else: + arch, arch_evidence = split_platform(plat_fact["value"] or "")[1], "python:sysconfig.get_platform" + if raw_gil_disabled in (0, 1): + free_threaded = observed(bool(raw_gil_disabled), "python:sysconfig.Py_GIL_DISABLED") + elif raw_gil_disabled is None and sys.version_info < (3, 13): + free_threaded = inferred(False, "python:sys.version_info") + else: + free_threaded = unknown("sysconfig gives no usable Py_GIL_DISABLED") + if hasattr(sys, "_is_gil_enabled"): + gil = observed(sys._is_gil_enabled(), "python:sys._is_gil_enabled") + elif free_threaded["value"] is False: + gil = inferred(True, *free_threaded["evidence"]) + else: + gil = unknown("sys._is_gil_enabled is missing") + try: + # Only glibc reports its version here. platform.libc_ver() would fall back to scanning + # the interpreter's executable, which gives the version it was linked against. + libc = os.confstr("CS_GNU_LIBC_VERSION") + except (AttributeError, ValueError, OSError): + libc = None + return { + "implementation": observed(sys.implementation.name, "python:sys.implementation.name"), + "version": observed(version, "python:platform.python_version"), + "cache_tag": observed(sys.implementation.cache_tag, "python:sys.implementation.cache_tag") if sys.implementation.cache_tag else unknown("no cache tag"), + "soabi": observed(soabi, "python:sysconfig.SOABI") if isinstance(soabi, str) and SLUG.fullmatch(soabi) else unknown(why if overrides else "sysconfig has no usable SOABI"), + "platform": plat_fact, + "arch": observed(arch, arch_evidence) if arch else unknown("the interpreter's build architecture was not found"), + "pointer_bits": observed(struct.calcsize("P") * 8, "python:struct.calcsize"), + "free_threaded_build": free_threaded, + "gil_enabled": gil, + "libc": observed(libc, "python:os.confstr") if isinstance(libc, str) and re.fullmatch(r"glibc [0-9]{1,4}\.[0-9]{1,4}(\.[0-9]{1,4})?", libc) else unknown("the C library reports no version through os.confstr; only glibc does"), + "execution": execution_fact(arch, arch_evidence, reference) if self.system == "Linux" else unknown("compared on Linux only"), + } + + def node(self, reference): + def all_unknown(reason): + return {name: unknown(reason) for name in ("version", "modules_abi", "napi_version", "platform", "arch", "execution")} + + if not self.probe_node: + return all_unknown("not probed (--no-node)") + # An unset PATH searches the default directories, but not the current directory that Windows lists first + # in them; an empty one finds nothing. + search_path = self.env("PATH") + if search_path is None: + search_path = os.pathsep.join(entry for entry in os.defpath.split(os.pathsep) if os.name != "nt" or entry != os.curdir) + executable = find_on_path("node", search_path) + if executable is None: + return all_unknown("node is not on PATH") + # Windows needs SystemRoot to start a program that loads side-by-side assemblies; nothing else is passed. + system_root = self.env("SystemRoot") if os.name == "nt" else None + try: + # On POSIX node gets a session, and so a process group, of its own; Windows ignores the option. + process = subprocess.Popen( + [executable, "-p", NODE_EXPRESSION], env={"SystemRoot": system_root} if system_root else {}, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, start_new_session=True, + ) + except OSError as exc: + return all_unknown(f"node could not be run ({type(exc).__name__})") + + def stop(): + # The whole group, so that a child holding stdout open cannot keep the read below waiting. + # A descendant that starts a session of its own can, until it exits or the job times out. + try: + if hasattr(os, "killpg"): + os.killpg(process.pid, signal.SIGKILL) + else: + process.kill() + except ProcessLookupError: + pass + + # Read at most one byte more than the limit, and stop node if it runs too long or writes too much. + # The timer stops node only while this thread reads. This thread ends that under the lock, so a + # timer that already fired has sent its signal by then, and only this thread reaps node: no signal + # can reach a process ID that was reaped and perhaps reused. + expired, lock = threading.Event(), threading.Lock() + deadline = time.monotonic() + NODE_TIMEOUT + reading = True + + def expire(): + with lock: + if reading: + expired.set() + stop() + + timer = threading.Timer(NODE_TIMEOUT, expire) + timer.start() + try: + with process.stdout: + output = process.stdout.read(NODE_OUTPUT_LIMIT + 1) + if len(output) > NODE_OUTPUT_LIMIT: + stop() + with lock: + reading = False + try: + status = process.wait(timeout=max(0.0, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + expired.set() + stop() + status = process.wait() + finally: + with lock: + reading = False + timer.cancel() + if process.poll() is None: + stop() + process.wait() + if expired.is_set(): + return all_unknown(f"node did not finish within {NODE_TIMEOUT} seconds") + if len(output) > NODE_OUTPUT_LIMIT: + return all_unknown(f"node wrote more than {NODE_OUTPUT_LIMIT} bytes") + if status != 0: + return all_unknown(f"node exited with status {status}") + values = json.loads(output.decode("utf-8")) + evidence = "cmd:node -p process.versions,process.arch,process.platform,os.endianness()" + + def checked(key, pattern): + value = values.get(key) + return observed(str(value), evidence) if value is not None and re.fullmatch(pattern, str(value)) else unknown(f"node reported no usable {key}") + + arch = normalize_arch(values.get("a")) + if arch == "ppc64" and values.get("e") == "LE": + # Node reports ppc64 for both byte orders; its own os.endianness() tells them apart. + arch = "ppc64le" + arch_evidence = [evidence] + arch_fact = observed(arch, evidence) if arch else unknown("node reported no usable architecture") + platform_name = values.get("p") + return { + "version": checked("v", r"[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}"), + "modules_abi": checked("m", r"[0-9]{1,5}"), + "napi_version": checked("n", r"[0-9]{1,5}"), + "platform": observed(platform_name, evidence) if isinstance(platform_name, str) and SLUG.fullmatch(platform_name) else unknown("node reported no usable platform"), + "arch": arch_fact, + "execution": execution_fact(arch, arch_evidence, reference) if self.system == "Linux" else unknown("compared on Linux only"), + } + + def run(self): + system = self.env("GITHUB_ACTIONS") == "true" + na = "not running in GitHub Actions" + + def gha(name, pattern): + return self.env_value(name, pattern) if system else not_applicable(na) + + def gha_ref(name, pattern): + # Python decodes the environment in the file system encoding, and git names refs in UTF-8: under another + # encoding a ref that is not ASCII would be recorded as other characters. + fact = gha(name, pattern) + if fact["value"] is not None and not fact["value"].isascii() and sys.getfilesystemencoding() != "utf-8": + return unknown(f"{name} is not ASCII, and the file system encoding is not UTF-8", f"env:{name}") + return fact + + ci = { + "system": observed("github-actions", "env:GITHUB_ACTIONS") if system else inferred("local", "env:GITHUB_ACTIONS"), + "event": gha("GITHUB_EVENT_NAME", r"[a-z_]{1,64}"), + "repository": gha("GITHUB_REPOSITORY", r"[A-Za-z0-9_.-]{1,100}/[A-Za-z0-9_.-]{1,100}"), + "ref": gha_ref("GITHUB_REF", GIT_REF), + "run_id": gha("GITHUB_RUN_ID", r"[0-9]{1,20}"), + "run_attempt": gha("GITHUB_RUN_ATTEMPT", r"[0-9]{1,10}"), + "run_url": not_applicable(na), + "workflow_ref": gha_ref("GITHUB_WORKFLOW_REF", WORKFLOW_REF), + "job": gha("GITHUB_JOB", r"[A-Za-z0-9_-]{1,100}"), + "runner_environment": gha("RUNNER_ENVIRONMENT", r"github-hosted|self-hosted"), + "runner_os": gha("RUNNER_OS", r"[A-Za-z0-9]{1,40}"), + "runner_arch": gha("RUNNER_ARCH", r"[A-Za-z0-9]{1,40}"), + "image_os": gha("ImageOS", r"[A-Za-z0-9._-]{1,40}"), + "image_version": gha("ImageVersion", r"[A-Za-z0-9._-]{1,40}"), + } + if ci["run_attempt"]["value"] is not None: + ci["run_attempt"]["value"] = int(ci["run_attempt"]["value"]) + # Up to the schema's largest integer, as the validator reads the attempt of a run URL. + if ci["run_attempt"]["value"] > 4294967295: + ci["run_attempt"] = unknown("GITHUB_RUN_ATTEMPT does not have the expected form", "env:GITHUB_RUN_ATTEMPT") + if system: + # The documented form; the attempt is in run_attempt. + server = self.env_value("GITHUB_SERVER_URL", r"https://[A-Za-z0-9.-]{1,100}") + parts = (server["value"], ci["repository"]["value"], ci["run_id"]["value"]) + if all(part is not None for part in parts): + ci["run_url"] = inferred(f"{parts[0]}/{parts[1]}/actions/runs/{parts[2]}", "env:GITHUB_SERVER_URL", "env:GITHUB_REPOSITORY", "env:GITHUB_RUN_ID") + else: + ci["run_url"] = unknown("a run URL needs GITHUB_SERVER_URL, GITHUB_REPOSITORY and GITHUB_RUN_ID") + revision = { + "tested_sha": self.checkout_sha(), + "event_sha": gha("GITHUB_SHA", r"[0-9a-f]{40}"), + } + revision.update(self.pull_request(ci["event"]["value"]) if system else {"pr_head_sha": not_applicable(na), "pr_base_sha": not_applicable(na)}) + return {"ci": ci, "revision": revision} + + def pull_request(self, event): + if event is None: + return {"pr_head_sha": unknown("the event is not known"), "pr_base_sha": unknown("the event is not known")} + if event not in ("pull_request", "pull_request_target"): + reason = f"the {event} event has no pull request" + return {"pr_head_sha": not_applicable(reason), "pr_base_sha": not_applicable(reason)} + path = self.env("GITHUB_EVENT_PATH") + if not path: + return {"pr_head_sha": unknown("GITHUB_EVENT_PATH is not set"), "pr_base_sha": unknown("GITHUB_EVENT_PATH is not set")} + try: + payload = json.loads(read_regular(path, 8 * 1024 * 1024).decode("utf-8")) + except (OSError, ValueError) as exc: + reason = f"the event file could not be read ({type(exc).__name__})" + return {"pr_head_sha": unknown(reason), "pr_base_sha": unknown(reason)} + # Only the two commit IDs are taken: the payload also holds logins, e-mail addresses and titles. + pull = payload.get("pull_request") if isinstance(payload, dict) else None + out = {} + for side in ("head", "base"): + sha = pull.get(side, {}).get("sha") if isinstance(pull, dict) and isinstance(pull.get(side), dict) else None + key = f"pr_{side}_sha" + out[key] = observed(sha, f"event:pull_request.{side}.sha") if isinstance(sha, str) and SHA.fullmatch(sha) else unknown(f"the event payload has no pull_request.{side}.sha") + return out + + def checkout_sha(self): + head, reason = self.read_git("HEAD") + if head is None: + return unknown(reason) + # Only ASCII blanks: a ref may end in another character str.strip() removes, such as U+3000. + head = head.strip(" \t\r\n") + if SHA.fullmatch(head): + return observed(head, "git:.git/HEAD") + ref = head[len("ref: "):] if head.startswith("ref: ") else None + # The ref is read as a path under .git, so it is followed only as git would name a ref: no ".." or "@{", and + # no part that is empty, starts with ".", or ends with ".lock" or "." (which Windows drops from a file name). + if ref is None or not re.fullmatch(GIT_REF, ref) or ".." in ref or "@{" in ref or any(not part or part.startswith(".") or part.endswith((".lock", ".")) for part in ref.split("/")): + return unknown(".git/HEAD has an unexpected form") + # Git names a ref's file in UTF-8, and the path is written in the file system encoding: under another encoding + # a name that is not ASCII could open another branch's file, and packed-refs alone could hold an older commit. + if not ref.isascii() and sys.getfilesystemencoding() != "utf-8": + return unknown("a ref that is not ASCII is followed only when the file system encoding is UTF-8") + # packed-refs is read only when there is no loose ref: git writes an update as a loose ref, so a loose ref that + # cannot be read, or holds no commit ID (such as a symbolic ref), may be newer than the packed one. + loose, reason = self.read_git(ref, absent_ok=True) + if reason is not None: + return unknown(reason) + if loose is not None: + if not SHA.fullmatch(loose.strip()): + return unknown(f".git/{ref} does not hold a commit ID") + # An evidence reference is printable ASCII, so a loose ref with another name is cited by .git/HEAD, which + # names it. + return observed(loose.strip(), f"git:.git/{ref}" if ref.isascii() else "git:.git/HEAD") + packed, _ = self.read_git("packed-refs") + # Lines end at "\n" only: str.splitlines() also ends one at characters a ref name may hold, such as U+2028. + for line in (packed or "").split("\n"): + sha, _, name = line.removesuffix("\r").partition(" ") + if name == ref and SHA.fullmatch(sha): + return observed(sha, "git:.git/packed-refs") + return unknown(f"{ref} is not resolved in .git") + + def read_git(self, name, absent_ok=False): + """(text, None), or (None, reason); with absent_ok, (None, None) when the file does not exist.""" + try: + if not self.git_dir.is_dir(): + return None, ".git is not a directory (no checkout, or a worktree)" + # A byte that is not UTF-8 becomes a lone surrogate, which no ref the collector follows holds; "replace" + # would turn it into U+FFFD, the name of another branch. + return read_regular(self.git_dir / name, 1 << 20).decode("utf-8", "surrogateescape"), None + except (OSError, ValueError) as exc: + if absent_ok and isinstance(exc, (FileNotFoundError, NotADirectoryError)): + return None, None + # ValueError: a name the file system encoding cannot write, which checkout_sha does not follow. + return None, f".git/{name} could not be read ({type(exc).__name__})" + + def compute(self): + return { + "backends": [{ + "id": "cpu", + "kind": "cpu", + "api": "cpu", + "available": observed(True, TOOL), + "device": unknown("the CPU model is not recorded"), + "version": not_applicable("a CPU has no backend version"), + "features": unknown("CPU features are not recorded"), + }], + "accelerator_inventory": unknown("this collector does not look for GPUs or NPUs; a missing entry does not show there is none"), + } + + def collect(self): + """Return the result document without stages; see build_result().""" + reason_layers = lambda reason: [layer_fallback(reason)] # noqa: E731 + reference = self.guard("cpu", lambda: self.cpu_reference() if self.system == "Linux" else (None, None), lambda reason: (None, None)) + return { + "run": self.guard("run", self.run, run_fallback), + "environment": {"layers": self.guard("layers", self.layers, reason_layers)}, + "workload": { + "identity": self.guard("identity", self.identity, identity_fallback), + "isolation": self.guard("isolation", self.isolation, isolation_fallback), + "runtimes": { + "python": self.guard("python", lambda: self.python(reference), python_fallback), + "node": self.guard("node", lambda: self.node(reference), node_fallback), + }, + }, + "compute": self.compute(), + } + + +def known_kind(layer): + fact = layer.get("kind", {}) + return fact.get("value") if fact.get("provenance") in ("observed", "inferred", "declared") else None + + +LAYER_ORDER = ("label", "kind", "technology", "os", "cpu", "service_manager", "container", "extensions") + + +def order_layer(layer): + layer["os"] = {key: layer["os"][key] for key in ("family", "name", "version", "kernel")} + return {key: layer[key] for key in LAYER_ORDER if key in layer} + + +def split_platform(plat): + """(OS family, architecture) from sysconfig.get_platform(), as python_platform_target() in validate_compatibility.py reads it.""" + if plat.startswith("linux-"): + return "linux", normalize_arch(plat[len("linux-"):]) + windows = {"win32": "x86", "win-amd64": "x86_64", "win-arm64": "aarch64", "win-arm32": "armv7"} + if plat in windows: + return "windows", windows[plat] + match = re.fullmatch(r"macosx-[0-9.]+-([A-Za-z0-9_]+)", plat) + if match: + return "macos", None if match.group(1) in ("universal2", "intel", "fat", "fat3", "fat64") else normalize_arch(match.group(1)) + match = re.fullmatch(r"freebsd-.+-([A-Za-z0-9_]+)", plat) + if match: + return "freebsd", normalize_arch(match.group(1)) + return None, None + + +def decode_capabilities(mask): + bits = int(mask, 16) + names = [f"cap_{name}" for index, name in enumerate(CAPABILITIES) if bits >> index & 1] + names += [f"cap_{index}" for index in range(len(CAPABILITIES), 64) if bits >> index & 1] + return names + + +def layer_fallback(reason): + return { + "kind": unknown(reason), "technology": unknown(reason), + "os": {"family": unknown(reason), "name": unknown(reason), "version": unknown(reason), "kernel": unknown(reason)}, + "cpu": {"arch": unknown(reason), "execution": unknown(reason)}, + "service_manager": {"name": unknown(reason), "scope": unknown(reason)}, + } + + +def identity_fallback(reason): + return { + "superuser": unknown(reason), "uid": unknown(reason), "real_uid": unknown(reason), "saved_uid": unknown(reason), + "gid": unknown(reason), "user_namespace": {"initial": unknown(reason), "uid_map": unknown(reason)}, + } + + +def isolation_fallback(reason): + return { + "no_new_privs": unknown(reason), + "capabilities": {"effective": unknown(reason), "permitted": unknown(reason), "bounding": unknown(reason)}, + "seccomp": unknown(reason), + "apparmor": {"enabled": unknown(reason), "profile": unknown(reason)}, + "selinux": {"mode": unknown(reason), "context": unknown(reason)}, + "cgroup": {"version": unknown(reason), "delegated": unknown(reason)}, + "pid_namespace": {"initial": unknown(reason)}, + } + + +def python_fallback(reason): + names = ("implementation", "version", "cache_tag", "soabi", "platform", "arch", "pointer_bits", "free_threaded_build", "gil_enabled", "libc", "execution") + return {name: unknown(reason) for name in names} + + +def node_fallback(reason): + return {name: unknown(reason) for name in ("version", "modules_abi", "napi_version", "platform", "arch", "execution")} + + +def run_fallback(reason): + names = ("system", "event", "repository", "ref", "run_id", "run_attempt", "run_url", "workflow_ref", "job", "runner_environment", "runner_os", "runner_arch", "image_os", "image_version") + return { + "ci": {name: unknown(reason) for name in names}, + "revision": {name: unknown(reason) for name in ("tested_sha", "event_sha", "pr_head_sha", "pr_base_sha")}, + } + + +def overall_of(stages): + statuses = [stages[name]["status"] for name in STAGES] + if "failed" in statuses: + return "failed" + if "passed" in statuses and all(status in ("passed", "not-applicable") for status in statuses): + return "passed" + return "incomplete" + + +def build_result(probe, profile_id=None, stage_args=None, now=None): + """Collect, then add the stages: detect from the probes, the others from stage_args.""" + sections = probe.collect() + if probe.errors: + names = ", ".join(error["probe"] for error in probe.errors) + detect = {"status": "failed", "reason": f"probe(s) raised unexpected errors: {names}; see probe_errors", "evidence": [f"{TOOL}#probe_errors"]} + else: + detect = {"status": "passed", "evidence": [TOOL]} + detect["privilege"] = {"escalation": declared("none", TOOL)} + stages = {"detect": detect} + for name in STAGES[1:]: + stages[name] = {"status": "not-run", "reason": "not run by the job that wrote this record"} + for name, stage in (stage_args or {}).items(): + stages[name] = stage + now = now or datetime.datetime.now(datetime.timezone.utc) + result = { + "schema_version": 1, + "kind": "result", + "synthetic": False, + "profile_id": profile_id, + "generated_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + "collector": dict(COLLECTOR), + "run": sections["run"], + "environment": sections["environment"], + "workload": sections["workload"], + "compute": sections["compute"], + "stages": stages, + "overall": overall_of(stages), + "probe_errors": probe.errors, + } + return result + + +def secret_shape(text): + """The label of the first secret shape that `text` matches, or None.""" + for label, shape in SECRET_SHAPES: + if shape.search(text): + return label + return None + + +def parse_stage_args(args): + """Turn the --stage* options into stage objects; raise ValueError on bad input.""" + def pairs(values, option): + for value in values or []: + name, sep, rest = value.partition("=") + if not sep or name not in STAGES[1:]: + raise ValueError(f"{option} takes NAME=VALUE with NAME one of {', '.join(STAGES[1:])}") + yield name, rest + + stages = {} + for name, status in pairs(args.stage, "--stage"): + if status not in STATUSES: + raise ValueError(f"--stage {name}: status must be one of {', '.join(STATUSES)}") + stages[name] = {"status": status} + for option, values in (("--stage-reason", args.stage_reason), ("--stage-evidence", args.stage_evidence), ("--stage-escalation", args.stage_escalation)): + for name, value in pairs(values, option): + if name not in stages: + raise ValueError(f"{option} {name}: give --stage {name}=STATUS first") + if secret_shape(value): + raise ValueError(f"{option} {name}: looks like a {secret_shape(value)}; records are published as CI artifacts") + if option == "--stage-reason": + if not re.fullmatch(r"[ -~]{1,500}", value): + raise ValueError(f"--stage-reason {name}: printable ASCII, 1 to 500 characters") + stages[name]["reason"] = value + elif option == "--stage-evidence": + if not EVIDENCE.fullmatch(value): + raise ValueError(f"--stage-evidence {name}: KIND:LOCATION with a known kind") + stages[name].setdefault("evidence", []).append(value) + else: + if value not in ESCALATIONS: + raise ValueError(f"--stage-escalation {name}: one of {', '.join(ESCALATIONS)}") + stages[name]["privilege"] = {"escalation": declared(value)} + return stages + + +def plain(text): + """Text for a terminal line, written as the validator's plain() writes it, so that no value (the output file's + name, for one) can start a GitHub Actions workflow command or show a value shaped like a secret. + """ + text = CONTROL.sub(" ", str(text)) + for label, shape in SECRET_SHAPES: + match = shape.search(text) + if match: + text = text[:match.start()] + f"[not shown: looks like a {label}]" + return text.replace("##[", "## [").replace("::", ": :") + + +def markdown_cell(value): + """Text for one Markdown table cell: control characters become spaces, and Markdown and HTML syntax is escaped.""" + return re.sub(r"([\\`*_~\[\]()<>!|&])", r"\\\1", re.sub(r"[\x00-\x1f\x7f-\x9f]", " ", str(value))) + + +def describe(fact): + if fact.get("provenance") in ("unknown", "not-applicable"): + return fact["provenance"] + value = fact["value"] + if isinstance(value, list): + value = f"{len(value)} item(s)" + return f"{value} ({fact['provenance']})" + + +def summary_lines(result): + ci, revision = result["run"]["ci"], result["run"]["revision"] + rows = [ + ("Profile", result["profile_id"] or "none"), + ("Event", describe(ci["event"])), + ("Run", describe(ci["run_url"])), + ("Tested commit (checkout)", describe(revision["tested_sha"])), + ("GITHUB_SHA", describe(revision["event_sha"])), + ("Pull request head / base", f"{describe(revision['pr_head_sha'])} / {describe(revision['pr_base_sha'])}"), + ] + for index, layer in enumerate(result["environment"]["layers"]): + os_facts = layer["os"] + rows.append((f"Layer {index}", f"{describe(layer['kind'])}; {describe(layer['technology'])}; {describe(os_facts['family'])} {describe(os_facts['name'])} {describe(os_facts['version'])}; kernel {describe(os_facts['kernel'])}; {describe(layer['cpu']['arch'])}, {describe(layer['cpu']['execution'])}")) + identity, isolation = result["workload"]["identity"], result["workload"]["isolation"] + rows += [ + ("Workload user", f"uid {describe(identity['uid'])}, real {describe(identity['real_uid'])}, saved {describe(identity['saved_uid'])}, superuser {describe(identity['superuser'])}, initial user namespace {describe(identity['user_namespace']['initial'])}"), + ("Isolation", f"no_new_privs {describe(isolation['no_new_privs'])}, seccomp {describe(isolation['seccomp'])}, effective capabilities {describe(isolation['capabilities']['effective'])}, permitted capabilities {describe(isolation['capabilities']['permitted'])}, AppArmor {describe(isolation['apparmor']['enabled'])}, SELinux {describe(isolation['selinux']['mode'])}, cgroup {describe(isolation['cgroup']['version'])}"), + ] + for name, runtime in result["workload"]["runtimes"].items(): + rows.append((name.capitalize(), ", ".join(f"{key} {describe(fact)}" for key, fact in runtime.items()))) + rows.append(("Stages", ", ".join(f"{name} {stage['status']}" for name, stage in result["stages"].items()))) + rows.append(("Overall", result["overall"])) + rows.append(("Probe errors", "; ".join(f"{error['probe']}: {error['error']}" for error in result["probe_errors"]) or "none")) + lines = ["### Compatibility record", "", "| Field | Value |", "|---|---|"] + lines += [f"| {markdown_cell(name)} | {markdown_cell(value)} |" for name, value in rows] + lines += ["", "Recorded by scripts/ci/collect_environment.py. Overall is incomplete while stages after detect are not run; see docs/CI-MATRIX.md.", ""] + return lines + + +class ArgumentParser(argparse.ArgumentParser): + def error(self, message): + # argparse repeats the argument it rejects, so its lines are written as every other line is. + usage = "".join(plain(line) + "\n" for line in self.format_usage().splitlines()) + self.exit(2, usage + plain(f"{self.prog}: error: {message}") + "\n") + + +def same_file(path, other): + """Whether two paths name one existing file, however each is written.""" + try: + return path.samefile(other) + except (OSError, ValueError): + return False + + +def main(argv=None): + parser = ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + parser.add_argument("--output", required=True, type=pathlib.Path, help="where to write the result JSON") + parser.add_argument("--profile-id", help="id of the profile this lane declares") + parser.add_argument("--summary", type=pathlib.Path, help="append a Markdown summary to this file") + parser.add_argument("--no-node", action="store_true", help="do not run node") + parser.add_argument("--stage", action="append", metavar="NAME=STATUS") + parser.add_argument("--stage-reason", action="append", metavar="NAME=TEXT") + parser.add_argument("--stage-evidence", action="append", metavar="NAME=REF") + parser.add_argument("--stage-escalation", action="append", metavar="NAME=VALUE") + args = parser.parse_args(argv) + if args.profile_id is not None and not SLUG.fullmatch(args.profile_id): + parser.error("--profile-id must be a lower-case identifier") + try: + stage_args = parse_stage_args(args) + except ValueError as exc: + parser.error(str(exc)) + + result = build_result(Probe(probe_node=not args.no_node), args.profile_id, stage_args) + text = json.dumps(result, indent=2, ensure_ascii=True) + "\n" + if secret_shape(text): + print(f"error: the record holds a value that looks like a {secret_shape(text)}; it was not written", file=sys.stderr) + return 2 + try: + args.output.write_text(text, encoding="utf-8") + except OSError as exc: + print(f"error: cannot write the record: {type(exc).__name__}", file=sys.stderr) + return 2 + if args.summary: + # Checked once the record exists, so that a link or another spelling of its name is found too. + if same_file(args.summary, args.output): + print("error: the record was written; --summary names the same file, so no summary was appended to it", file=sys.stderr) + return 2 + try: + with open(args.summary, "a", encoding="utf-8") as handle: + handle.write("\n".join(summary_lines(result))) + except OSError as exc: + print(f"error: the record was written, but the summary could not be: {type(exc).__name__}", file=sys.stderr) + return 2 + detect = result["stages"]["detect"]["status"] + print(plain(f"{args.output.name}: detect {detect}, overall {result['overall']}, {len(result['probe_errors'])} probe error(s)")) + for error in result["probe_errors"]: + print(plain(f" probe error: {error['probe']}: {error['error']}")) + return 0 if detect == "passed" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/tests/test_compatibility_schema.py b/scripts/ci/tests/test_compatibility_schema.py new file mode 100644 index 0000000..47641aa --- /dev/null +++ b/scripts/ci/tests/test_compatibility_schema.py @@ -0,0 +1,2110 @@ +"""Tests for scripts/ci/validate_compatibility.py and tests/compatibility/schema.json. + +The committed profiles and the synthetic fixtures must be valid. Every other +test changes one field of a valid document and checks the exact error code the +change must produce, so a rule that stops firing fails here. No Django and no +third-party package is needed. From the repository root: + python -I scripts/ci/tests/test_compatibility_schema.py --verbose +""" + +import contextlib +import copy +import hashlib +import importlib.util +import io +import json +import pathlib +import re +import sys +import tempfile +import time +import unittest +from unittest import mock + +ROOT = pathlib.Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "ci" / "validate_compatibility.py" +COMPATIBILITY = ROOT / "tests" / "compatibility" +FIXTURES = COMPATIBILITY / "fixtures" +STRATEGY = ROOT / "docs" / "DEPENDENCY-STRATEGY.md" + +spec = importlib.util.spec_from_file_location("validate_compatibility", SCRIPT) +validator = importlib.util.module_from_spec(spec) +spec.loader.exec_module(validator) + +SCHEMA = validator.load_schema() +LANES = validator.read_strategy_lanes(STRATEGY) +PAIRED = ("linux-x64-vm", "windows-arm64-native", "windows-arm64-wsl2-podman", "kubernetes-arm64-cuda") + + +def fixture(name): + return validator.load_json(FIXTURES / f"{name}.json") + + +def pair(name): + return fixture(f"{name}.result"), fixture(f"{name}.profile") + + +def check(document, profile=None, lanes=LANES, policy=True, real=False): + return validator.validate_document(document, SCHEMA, profile, lanes, policy, real) + + +def changed(document, change): + document = copy.deepcopy(document) + change(document) + return document + + +def unknown(reason="not recorded"): + return {"value": None, "provenance": "unknown", "reason": reason} + + +def fact(value, *evidence, provenance="observed"): + out = {"value": value, "provenance": provenance} + if evidence: + out["evidence"] = list(evidence) + return out + + +def real_result(): + """A measured-looking result built from the Linux fixture: the shape the collector writes on a runner.""" + result, profile = pair("linux-x64-vm") + + def unmark(value): + if isinstance(value, dict): + if isinstance(value.get("evidence"), list): + value["evidence"] = [ref.replace("synthetic:", "file:", 1) for ref in value["evidence"]] + for item in value.values(): + unmark(item) + elif isinstance(value, list): + for item in value: + unmark(item) + + result, profile = copy.deepcopy(result), copy.deepcopy(profile) + unmark(result) + result["synthetic"] = False + result["generated_at"] = "2026-10-11T00:00:00Z" + result["profile_id"] = profile["id"] = "linux-x64-vm" + result.pop("extensions") + result["stages"]["detect"]["evidence"] = ["tool:collect-environment"] + sha = "0123456789abcdef0123456789abcdef01234567" + result["run"] = { + "ci": { + "system": fact("github-actions", "env:GITHUB_ACTIONS"), + "event": fact("pull_request", "env:GITHUB_EVENT_NAME"), + "repository": fact("P-R-N-D/Claus-AI", "env:GITHUB_REPOSITORY"), + "ref": fact("refs/pull/1/merge", "env:GITHUB_REF"), + "run_id": fact("100", "env:GITHUB_RUN_ID"), + "run_attempt": fact(1, "env:GITHUB_RUN_ATTEMPT"), + "run_url": fact("https://github.com/P-R-N-D/Claus-AI/actions/runs/100", "env:GITHUB_RUN_ID", provenance="inferred"), + "workflow_ref": fact("P-R-N-D/Claus-AI/.github/workflows/ci.yml@refs/pull/1/merge", "env:GITHUB_WORKFLOW_REF"), + "job": fact("compatibility", "env:GITHUB_JOB"), + "runner_environment": fact("github-hosted", "env:RUNNER_ENVIRONMENT"), + "runner_os": fact("Linux", "env:RUNNER_OS"), + "runner_arch": fact("X64", "env:RUNNER_ARCH"), + "image_os": fact("ubuntu24", "env:ImageOS"), + "image_version": fact("20261004.327.1", "env:ImageVersion"), + }, + "revision": { + "tested_sha": fact(sha, "git:.git/HEAD"), + "event_sha": fact(sha, "env:GITHUB_SHA"), + "pr_head_sha": fact("1" * 40, "event:pull_request.head.sha"), + "pr_base_sha": fact("2" * 40, "event:pull_request.base.sha"), + }, + } + profile["synthetic"] = False + profile["support"]["status"] = "checked" + # The lane is checked for glibc 2.39 or newer, so a checked profile declares the glibc version its run measures. + profile["workload"]["runtimes"]["python"]["libc"] = fact("glibc 2.39", provenance="declared") + profile["evidence_sources"] = [{"workflow": ".github/workflows/ci.yml", "job": "compatibility"}] + profile.pop("extensions") + return result, profile + + +def macos_profile(): + """The real profile of real_result(), moved to an Apple Silicon Mac: one host layer running macOS.""" + _, profile = real_result() + layers = profile["environment"]["layers"] + host = layers[0] + host["os"].update(family=fact("macos", provenance="declared"), name=unknown(), version=unknown(), kernel=unknown()) + host["cpu"].update(arch=fact("aarch64", provenance="declared"), execution=unknown()) + host["service_manager"].update(name=unknown(), scope=unknown()) + profile["environment"]["layers"] = [host] + profile["workload"]["runtimes"]["python"].update( + platform=fact("macosx-15.0-arm64", provenance="declared"), arch=fact("aarch64", provenance="declared"), + soabi=fact("cpython-312-darwin", provenance="declared"), libc=unknown(), execution=unknown()) + profile["workload"]["runtimes"]["node"].update(platform=fact("darwin", provenance="declared"), arch=fact("aarch64", provenance="declared"), execution=unknown()) + return profile + + +class Case(unittest.TestCase): + def assertCodes(self, findings, *codes, level="error"): + self.assertEqual(findings.codes(level), sorted(codes), findings.items) + + def assertValid(self, findings): + self.assertEqual(findings.errors, [], findings.items) + + def assertFinding(self, findings, code, path, level="error"): + """The finding with this code is reported at this path, not only somewhere.""" + found = [(code_, path_) for kind, code_, path_, _ in findings.items if kind == level] + self.assertIn((code, path), found, findings.items) + + +class CommittedDocumentTests(Case): + def test_paired_fixtures_are_valid(self): + for name in PAIRED: + result, profile = pair(name) + with self.subTest(name=name): + self.assertValid(check(profile)) + self.assertValid(check(result, profile)) + + def test_unpaired_fixture_is_valid_under_the_default_policy(self): + self.assertValid(check(fixture("vm-podman-quadlet-systemd.result"))) + + def test_the_rootful_cluster_fixture_only_warns_about_the_preferred_rule(self): + result, profile = pair("kubernetes-arm64-cuda") + findings = check(result, profile) + self.assertCodes(findings) + self.assertCodes(findings, "policy-engine-rootless", "policy-exception-applied", level="warning") + applied = [message for kind, code, _, message in findings.items if code == "policy-exception-applied"] + self.assertEqual(len(applied), 1, findings.items) + self.assertIn("gpu-device (minimal-devices) covers devices nvidia-gpu", applied[0]) + self.assertIn(profile["policy"]["exceptions"][0]["approval"]["reference"], applied[0]) + + def test_fixtures_are_marked_synthetic_and_carry_no_measured_data(self): + for path in sorted(FIXTURES.glob("*.json")): + document = validator.load_json(path) + with self.subTest(path=path.name): + self.assertIs(document["synthetic"], True) + refs = [ref for _, ref in validator.evidence_refs(document)] + self.assertTrue(all(ref.startswith("synthetic:") for ref in refs)) + if document["kind"] == "profile": + self.assertTrue(document["id"].startswith("synthetic-")) + self.assertNotEqual(document["support"]["status"], "checked") + else: + self.assertTrue(refs) + self.assertIsNone(document["generated_at"]) + self.assertTrue(all(value["provenance"] == "not-applicable" for value in document["run"]["ci"].values())) + + def test_committed_profiles_are_real_and_linked_to_a_job(self): + profiles = sorted((COMPATIBILITY / "profiles").glob("*.json")) + self.assertTrue(profiles) + for path in profiles: + profile = validator.load_json(path) + with self.subTest(path=path.name): + self.assertIs(profile["synthetic"], False) + self.assertTrue(profile["evidence_sources"]) + self.assertValid(check(profile)) + + def test_the_real_profile_declares_its_runner(self): + profile = validator.load_json(COMPATIBILITY / "profiles" / "linux-x64-gha.json") + self.assertEqual({name: fact["value"] for name, fact in profile["runner"].items()}, + {"runner_environment": "github-hosted", "runner_os": "Linux", "runner_arch": "X64", "image_os": "ubuntu24"}) + + def test_required_scenarios_are_covered(self): + documents = [validator.load_json(path) for path in FIXTURES.glob("*.result.json")] + kinds = [[layer["kind"]["value"] for layer in validator.layers_of(document)] for document in documents] + self.assertIn(["host"], kinds) + self.assertIn(["host", "wsl2", "container"], kinds) + self.assertIn(["host", "vm", "container"], kinds) + self.assertIn(["host", "container"], kinds) + apis = {backend["api"] for document in documents for backend in document["compute"]["backends"]} + self.assertTrue({"cpu", "cuda", "qnn"} <= apis) + wsl = fixture("windows-arm64-wsl2-podman.result") + self.assertEqual(wsl["environment"]["layers"][2]["container"]["engine_mode"]["value"], "rootless") + self.assertEqual(wsl["workload"]["identity"]["uid"]["value"], 1000) + quadlet = fixture("vm-podman-quadlet-systemd.result")["environment"]["layers"] + self.assertEqual(quadlet[1]["service_manager"]["name"]["value"], "systemd") + self.assertEqual(quadlet[2]["service_manager"]["name"]["value"], "systemd") + self.assertEqual(quadlet[2]["container"]["orchestration"]["scope"]["value"], "user") + + +class SchemaSubsetTests(Case): + def keywords(self, node, out): + if isinstance(node, dict): + out.update(node) + for key in ("properties", "$defs"): + for sub in node.get(key, {}).values(): + self.keywords(sub, out) + for key in ("items", "additionalProperties", "propertyNames", "if", "then", "else"): + if key in node: + self.keywords(node[key], out) + return out + + def test_the_schema_uses_only_the_supported_keywords(self): + used = self.keywords(SCHEMA, set()) + self.assertLessEqual(used, validator.SUPPORTED_KEYWORDS) + self.assertEqual(used & validator.CHECKING_KEYWORDS, validator.CHECKING_KEYWORDS - {"minLength", "minimum"} | (used & {"minLength", "minimum"})) + + def test_other_keywords_are_rejected_before_any_document_is_read(self): + for keyword in ("oneOf", "anyOf", "allOf", "not", "format", "uniqueItems", "patternProperties", + "exclusiveMinimum", "multipleOf", "dependentRequired", "prefixItems", "contains", + "unevaluatedProperties", "$dynamicRef", "default"): + with self.subTest(keyword=keyword), self.assertRaises(validator.SchemaError): + validator.check_schema({"type": "object", "properties": {"a": {keyword: []}}}) + for bad in ( + {"$defs": {"a": {"$defs": {}}}, "$ref": "#/$defs/a"}, + {"$ref": "#/$defs/missing"}, + {"$ref": "other.json#/x"}, + {"pattern": "("}, + {"type": "decimal"}, + {"minItems": -1}, + {"maxLength": 1.0}, + {"required": "a"}, + {"$schema": "http://json-schema.org/draft-07/schema#"}, + {"properties": {"a": {"$schema": validator.DIALECT}}}, + {"properties": {"a": {"$id": "https://example.invalid/a"}}}, + {"$id": 1}, + {"title": ["a"]}, + {"description": 1}, + {"type": [1]}, + {"type": [["string"]]}, + {"pattern": "^[0-9]\\d$"}, + {"pattern": "\\w+"}, + {"pattern": "\\bword"}, + {"pattern": "a\\Z"}, + {"pattern": "\\Aa"}, + {"pattern": "(?i)a"}, + {"pattern": "(?Pa)"}, + {"pattern": "^a{,3}$"}, + {"pattern": "^a*+b$"}, + {"pattern": "^a{2}+$"}, + {"pattern": "^\\a$"}, + {"pattern": "^\\N{LATIN SMALL LETTER A}$"}, + {"pattern": "^\\U00000061$"}, + {"pattern": "^\\uD83D\\uDE00$"}, + {"pattern": "^[\\udc00-\\uDFFF]$"}, + {"pattern": "^[]a]$"}, + {"pattern": "^[^]a]$"}, + {"$defs": {"a": {"if": {"$ref": "#/$defs/a"}}}, "$ref": "#/$defs/a"}, + {"$defs": {"a": {"$ref": "#/$defs/b"}, "b": {"else": {"$ref": "#/$defs/a"}}}}, + ): + with self.subTest(schema=bad), self.assertRaises(validator.SchemaError): + validator.check_schema(bad) + for bad in ([], "schema", True): + with self.subTest(root=bad), self.assertRaises(validator.SchemaError): + validator.check_schema(bad) + for good in ( + {"$schema": validator.DIALECT, "$id": "https://example.invalid/s", "title": "t", "$comment": "c"}, + {"pattern": "^(?:a|b)[0-9]\\\\d$"}, + {"pattern": "^\\*+a{0,3}$"}, + {"pattern": "^\\t\\n\\x41\\u0041\\.\\[\\]$"}, + {"$defs": {"a": {"properties": {"x": {"$ref": "#/$defs/a"}}, "items": {"$ref": "#/$defs/a"}}}, "$ref": "#/$defs/a"}, + ): + with self.subTest(schema=good): + validator.check_schema(good) + + def test_each_checking_keyword_is_applied(self): + defs = {"$defs": {"s": {"type": "string"}}} + cases = [ + ({"type": "integer"}, 1, 1.5), + ({"type": "integer"}, 1, True), + ({"type": "integer"}, 1, 1.0), + ({"type": "number"}, 1, "1"), + ({"type": ["string", "null"]}, None, 0), + ({"enum": [1, "a"]}, "a", True), + ({"const": False}, False, 0), + ({"properties": {"a": {"type": "string"}}}, {"a": "x"}, {"a": 1}), + ({"properties": {"a": False, "b": True}}, {"b": 1}, {"a": 1}), + ({"required": ["a"]}, {"a": 1}, {"b": 1}), + ({"additionalProperties": False, "properties": {"a": {}}}, {"a": 1}, {"a": 1, "b": 2}), + ({"additionalProperties": {"type": "integer"}}, {"b": 2}, {"b": "2"}), + ({"propertyNames": {"pattern": "^x-"}}, {"x-a": 1}, {"a": 1}), + ({"items": {"type": "integer"}}, [1, 2], [1, "2"]), + ({"minItems": 1}, [1], []), + ({"maxItems": 1}, [1], [1, 2]), + ({"minLength": 2}, "ab", "a"), + ({"maxLength": 2}, "ab", "abc"), + ({"pattern": "^a+$"}, "aa", "ab"), + ({"minimum": 0}, 0, -1), + ({"maximum": 10}, 10, 11), + (dict(defs, **{"$ref": "#/$defs/s"}), "a", 1), + ({"if": {"properties": {"k": {"const": 1}}}, "then": {"required": ["a"]}, "else": {"required": ["b"]}}, {"k": 1, "a": 0}, {"k": 1, "b": 0}), + ({"if": {"properties": {"k": {"const": 1}}}, "then": {"required": ["a"]}, "else": {"required": ["b"]}}, {"k": 2, "b": 0}, {"k": 2, "a": 0}), + ] + exercised = set() + for schema, good, bad in cases: + validator.check_schema(schema) + exercised.update(set(schema) & validator.CHECKING_KEYWORDS) + with self.subTest(schema=schema, value=good): + findings = validator.Findings() + validator.validate_schema(good, schema, schema, "", findings) + self.assertEqual(findings.items, []) + with self.subTest(schema=schema, value=bad): + findings = validator.Findings() + validator.validate_schema(bad, schema, schema, "", findings) + self.assertTrue(findings.errors) + self.assertEqual(exercised, validator.CHECKING_KEYWORDS) + + def test_a_structural_error_stops_the_cross_field_rules(self): + result = changed(fixture("linux-x64-vm.result"), lambda d: d["stages"].pop("security")) + findings = check(result) + self.assertCodes(findings, "schema") + self.assertCodes(findings, "semantic-skipped", level="warning") + + def test_structure_errors(self): + result = fixture("linux-x64-vm.result") + for label, change in ( + ("missing field", lambda d: d["workload"].pop("identity")), + ("wrong type", lambda d: d["workload"]["identity"]["uid"].update(value="1001")), + ("unknown status", lambda d: d["stages"]["install"].update(status="skipped")), + ("unknown provenance", lambda d: d["workload"]["identity"]["uid"].update(provenance="guessed")), + ("schema version", lambda d: d.update(schema_version=2)), + ("extra field", lambda d: d.update(notes="x")), + ("extension without x-", lambda d: d["extensions"].update(note="x")), + ("layer field without x-", lambda d: d["environment"]["layers"][1].update(cloud="azure")), + ("too many layers", lambda d: d["environment"].update(layers=d["environment"]["layers"] * 5)), + ("bad evidence kind", lambda d: d["stages"]["detect"].update(evidence=["https://example.com"])), + ("bad sha", lambda d: d["run"]["revision"].update(tested_sha=fact("ABC", "synthetic:x"))), + ): + with self.subTest(label): + self.assertCodes(check(changed(result, change)), "schema") + + def test_extension_fields_are_accepted_and_ignored(self): + result = changed(fixture("linux-x64-vm.result"), lambda d: ( + d["extensions"].update({"x-vendor-data": {"anything": [1, 2]}}), + d["environment"]["layers"][1].update(extensions={"x-azure-sku": "Standard_D4ads_v6"}), + d["stages"]["detect"].update(extensions={"x-duration-seconds": 3}), + )) + self.assertValid(check(result)) + # Contents that look like facts or evidence are not facts: only strings are checked there. + for label, data in ( + ("unknown fact without a reason", {"value": None, "provenance": "unknown"}), + ("measured evidence in a synthetic document", {"value": "x", "provenance": "observed", "evidence": ["file:/x"]}), + ("evidence that is not a string", {"evidence": [1]}), + ): + with self.subTest(label): + self.assertValid(check(changed(result, lambda d: d["extensions"].update({"x-probe": data})))) + + +class JsonLoadingTests(Case): + def write(self, directory, content): + path = pathlib.Path(directory) / "document.json" + path.write_bytes(content) + return path + + def test_lenient_json_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + for label, content in ( + ("duplicate key", b'{"a": 1, "a": 2}'), + ("NaN", b'{"a": NaN}'), + ("Infinity", b'{"a": -Infinity}'), + ("overflow", b'{"a": 1e999}'), + ("byte order mark", b'\xef\xbb\xbf{}'), + ("not UTF-8", b'{"a": "\xff"}'), + ("deep nesting", b"[" * 100000 + b"]" * 100000), + ("too large", b'{"a": "' + b"x" * validator.MAX_DOCUMENT_BYTES + b'"}'), + ("integer outside the double range", b'{"a": 1' + b"0" * 400 + b"}"), + ): + with self.subTest(label), self.assertRaises(ValueError): + validator.load_json(self.write(directory, content)) + + def test_a_document_is_read_only_up_to_the_size_limit(self): + handle = mock.MagicMock() + handle.__enter__.return_value.read.return_value = b" " * (validator.MAX_DOCUMENT_BYTES + 1) + with mock.patch.object(validator, "open", create=True, return_value=handle), self.assertRaisesRegex(ValueError, "larger than"): + validator.load_json("large.json") + handle.__enter__.return_value.read.assert_called_once_with(validator.MAX_DOCUMENT_BYTES + 1) + + def test_the_command_reports_unparsable_documents_as_invalid(self): + with tempfile.TemporaryDirectory() as directory: + path = self.write(directory, b'{"a": 1, "a": 2}') + with contextlib.redirect_stdout(io.StringIO()) as output: + self.assertEqual(validator.main([str(path)]), 1) + self.assertIn("ERROR json /", output.getvalue()) + + +class ProvenanceTests(Case): + def test_fact_rules(self): + result = fixture("linux-x64-vm.result") + uid = ("workload", "identity", "uid") + + def set_uid(value): + return lambda d: d["workload"]["identity"].update(uid=value) + + for label, change in ( + ("observed without evidence", set_uid({"value": 1001, "provenance": "observed"})), + ("observed without value", set_uid({"value": None, "provenance": "observed", "evidence": ["synthetic:x"]})), + ("inferred without value", set_uid({"value": None, "provenance": "inferred", "evidence": ["synthetic:x"]})), + ("inferred without evidence", set_uid({"value": 1001, "provenance": "inferred", "evidence": []})), + ("declared without value", set_uid({"value": None, "provenance": "declared"})), + ("unknown with a value", set_uid({"value": 1001, "provenance": "unknown", "reason": "x"})), + ("unknown without reason", set_uid({"value": None, "provenance": "unknown"})), + ("not-applicable with a value", set_uid({"value": 1001, "provenance": "not-applicable", "reason": "x"})), + ): + with self.subTest(label): + findings = check(changed(result, change), policy=False) + self.assertIn("fact-provenance", findings.codes(), uid) + + def test_a_profile_cannot_hold_observations(self): + for provenance in ("observed", "inferred"): + with self.subTest(provenance): + profile = changed(fixture("linux-x64-vm.profile"), lambda d: d["environment"]["layers"][1]["os"].update( + name=fact("ubuntu", "synthetic:x", provenance=provenance))) + self.assertCodes(check(profile), "profile-provenance") + + def test_synthetic_marking(self): + result = fixture("linux-x64-vm.result") + profile = fixture("linux-x64-vm.profile") + cases = ( + ("measured evidence in a synthetic document", result, lambda d: d["stages"]["detect"].update(evidence=["log:job"])), + ("synthetic result with a collection time", result, lambda d: d.update(generated_at="2026-10-11T00:00:00Z")), + ("synthetic result claiming a run", result, lambda d: d["run"]["ci"].update(system=fact("github-actions", "synthetic:x"))), + ("synthetic result naming a run URL", result, lambda d: d["run"]["ci"].update(system=unknown(), run_url=fact("https://github.com/P-R-N-D/Claus-AI/actions/runs/1", provenance="declared"))), + ("synthetic result naming a commit", result, lambda d: d["run"]["revision"].update(event_sha=fact("1" * 40, provenance="declared"))), + ("real result without a collection time", result, lambda d: d.update(synthetic=False)), + ("synthetic profile without the id prefix", profile, lambda d: d.update(id="linux-x64-vm")), + ) + for label, document, change in cases: + with self.subTest(label): + self.assertIn("synthetic-marking", check(changed(document, change), policy=False).codes()) + measured, _ = real_result() + self.assertCodes(check(changed(measured, lambda d: d["stages"]["detect"]["evidence"].append("synthetic:x"))), "synthetic-marking") + self.assertCodes(check(result, real=True), "synthetic-marking") + undated = check(changed(measured, lambda d: d.update(generated_at=None))) + self.assertCodes(undated, "synthetic-marking") + self.assertFinding(undated, "synthetic-marking", "/generated_at") + # The collection time has the schema's shape and is also a day and time of day that exist. + for generated_at in ("2026-99-99T99:99:99Z", "2026-02-30T12:00:00Z", "2025-02-29T12:00:00Z", "2026-10-11T24:00:00Z", "2026-10-11T23:60:00Z", "2026-10-11T23:59:60Z"): + with self.subTest(generated_at=generated_at): + impossible = check(changed(measured, lambda d: d.update(generated_at=generated_at))) + self.assertCodes(impossible, "synthetic-marking") + self.assertFinding(impossible, "synthetic-marking", "/generated_at") + self.assertValid(check(changed(measured, lambda d: d.update(generated_at="2024-02-29T23:59:59Z")))) + + def test_strings_with_control_characters_or_secrets_are_rejected(self): + result = fixture("linux-x64-vm.result") + for label, text, code in ( + ("newline", "a\nb", "control-character"), + ("C1 control sequence introducer", "a\x9b31mb", "control-character"), + ("line separator", "a\u2028b", "control-character"), + ("bidirectional override", "a\u202eb", "control-character"), + ("lone surrogate", "a\ud800b", "control-character"), + ("GitHub token", "token ghp_" + "A" * 36, "secret-like-value"), + ("fine-grained token", "github_pat_" + "a" * 30, "secret-like-value"), + ("AWS key", "AKIA" + "ABCDEFGHIJKLMNOP", "secret-like-value"), + # A key ID ends where its alphabet does, so a word character after it does not hide it. + ("AWS key before an underscore", "AKIA" + "ABCDEFGHIJKLMNOP" + "_suffix", "secret-like-value"), + ("AWS key before a lowercase letter", "ASIA" + "ABCDEFGHIJKLMNOP" + "abc", "secret-like-value"), + ("run of 17 key characters", "AKIA" + "ABCDEFGHIJKLMNOPQ", None), + ("private key", "-----BEGIN OPENSSH PRIVATE KEY-----", "secret-like-value"), + ("JSON Web Token after a letter", "xeyJ" + "A" * 10 + ".eyJ" + "B" * 10 + ".", "secret-like-value"), + # The shortest JSON Web Token shape has 8 characters after each "eyJ"; one fewer on either side is not one. + ("shortest JSON Web Token", "eyJ" + "A" * 8 + ".eyJ" + "B" * 8 + ".", "secret-like-value"), + ("shortest JSON Web Token after a letter", "xeyJ" + "A" * 8 + ".eyJ" + "B" * 8 + ".", "secret-like-value"), + ("short first part", "eyJ" + "A" * 7 + ".eyJ" + "B" * 8 + ".", None), + ("short second part", "eyJ" + "A" * 8 + ".eyJ" + "B" * 7 + ".", None), + ): + with self.subTest(label): + document = changed(result, lambda d: d["stages"]["install"].update(reason=text)) + self.assertCodes(check(document), *([code] if code else [])) + document = changed(result, lambda d: d["extensions"].update({"x-data": {"ghp_" + "A" * 36: 1}})) + self.assertCodes(check(document), "secret-like-value") + self.assertNotIn("ABCDEFGHIJKLMNOP", validator.plain("key AKIA" + "ABCDEFGHIJKLMNOP" + "_suffix")) + # A sample for every shape, found on its own and right after a letter, also one outside ASCII. + samples = ("ghp_" + "A" * 36, "github_pat_" + "a" * 30, "AKIA" + "ABCDEFGHIJKLMNOP", "-----BEGIN OPENSSH PRIVATE KEY-----", "xoxb-" + "A" * 12, "xapp-1-" + "A" * 12, "eyJ" + "A" * 10 + ".eyJ" + "B" * 10 + ".") + for label, shape in validator.SECRET_SHAPES: + self.assertTrue(any(shape.search(sample) for sample in samples), f"no sample for {label}") + for sample in samples: + for before in ("", "x", "\u00e9"): + with self.subTest(sample=sample[:6], before=before): + document = changed(result, lambda d: d["stages"]["install"].update(reason=before + sample)) + self.assertCodes(check(document), "secret-like-value") + # A long run with many "eyJ" in it is searched in linear time; a quadratic search takes seconds here. + started = time.perf_counter() + for text in ("eyJ" * 40000, "-eyJ" * 30000, "eyJ" * 20000 + "." + "eyJ" * 20000): + self.assertFalse([label for label, shape in validator.SECRET_SHAPES if shape.search(text)]) + self.assertLess(time.perf_counter() - started, 2) + + def test_a_long_key_or_deep_nesting_does_not_lengthen_every_path(self): + # Every path below a key repeats it, so a long key or deep nesting over many values would make as many + # long paths, and the walk built a container's child paths all at once, before it visited any of them. + def named(text): + digest = hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest()[:8] + return f"~({len(text)} characters, sha256 {digest})" + + # The limit is on the escaped key, and a replaced key keeps the first 8 hex digits of its digest, so keys of + # one length are told apart unless those digits agree. An escaped key holds "~" only before "0" or "1", so a + # key that looks like a marker is not one. + kept = ("k" * 128, "~" * 64, "/" * 64, "...", "~...", "(5 characters)") + self.assertEqual([validator.pointer("", key) for key in kept], ["/" + "k" * 128, "/" + "~0" * 64, "/" + "~1" * 64, "/...", "/~0...", "/(5 characters)"]) + replaced = ("k" * 129, "~" * 65, "/" * 65, "x" * 200, "y" * 200, "\ud800" * 200) + self.assertEqual([validator.pointer("", key) for key in replaced], ["/" + named(key) for key in replaced]) + self.assertEqual(len({validator.pointer("", key) for key in replaced}), len(replaced)) + sixteen = "/" + "/".join(str(index) for index in range(15)) + self.assertEqual((validator.pointer(sixteen, 15), validator.pointer(validator.pointer(sixteen, 15), 16)), + (sixteen + "/15", "/0/1/2/3/4/5/6/7/~.../10/11/12/13/14/15/16")) + result = fixture("linux-x64-vm.result") + long = check(changed(result, lambda d: d["extensions"].update({"x-a": {"k" * 100000: ["\u0001"] * 1000}}))) + paths = [path for _, code, path, _ in long.items if code == "control-character"] + # Sets, so that a failure lists the paths that differ rather than diffing 1000 similar lines. + self.assertEqual(len(paths), 1000) + self.assertLessEqual(max(map(len, paths)), validator.PATH_KEYS * (validator.KEY_LIMIT + 1)) + self.assertEqual(set(paths), {f"/extensions/x-a/{named('k' * 100000)}/{index}" for index in range(1000)}) + keys = [f"a{index}" for index in range(40)] + nested = "\u0001" + for key in reversed(keys): + nested = {key: nested} + deep = check(changed(result, lambda d: d["extensions"].update({"x-a": nested}))) + self.assertFinding(deep, "control-character", "/" + "/".join(["extensions", "x-a", *keys[:6], "~...", *keys[-7:]])) + # However long and deep the keys, a path keeps at most PATH_KEYS keys of at most KEY_LIMIT characters. The + # chain keys are kept at 127 characters once escaped, or are under KEY_LIMIT characters but over it once + # escaped, and a finding sits under every long key. + for width in (63, 100): + chain = "\u0001" + for index in range(40): + chain = {"~" * width + chr(ord("a") + index % 26): chain, "~" * 200: "\u0001"} + with self.subTest(width=width): + bounded = [path.split("/")[1:] for _, _, path, _ in check(changed(result, lambda d: d["extensions"].update({"x-a": chain}))).items] + self.assertTrue(bounded) + self.assertLessEqual(max(map(len, bounded)), validator.PATH_KEYS) + self.assertLessEqual(max(len(key) for keys in bounded for key in keys), validator.KEY_LIMIT) + # A value's path is built when the value is reached. + built = [] + original = validator.pointer + with mock.patch.object(validator, "pointer", side_effect=lambda *args: built.append(args) or original(*args)): + values = validator.walk({"a": list(range(1000))}) + self.assertEqual([path for path, _ in (next(values) for _ in range(3))], ["", "/a", "/a/999"]) + self.assertEqual(len(built), 2) + + +class EnvironmentRuleTests(Case): + def test_layer_structure(self): + result = fixture("windows-arm64-wsl2-podman.result") + layers = lambda d: d["environment"]["layers"] # noqa: E731 + at = "/environment/layers/" + for label, change, code, path in ( + ("host inside", lambda d: layers(d)[1].update(kind=fact("host", "synthetic:x")), "layer-structure", at + "1/kind"), + ("WSL2 outermost", lambda d: layers(d).pop(0), "layer-structure", at + "0/kind"), + ("container outermost", lambda d: d["environment"].update(layers=layers(d)[2:]), "layer-structure", at + "0/kind"), + ("VM outermost", lambda d: (layers(d)[1].update(kind=fact("vm", "synthetic:x"), technology=fact("hyper-v", "synthetic:x")), layers(d).pop(0)), "layer-structure", at + "0/kind"), + ("container without container facts", lambda d: layers(d)[2].pop("container"), "layer-structure", at + "2"), + ("container facts on a VM", lambda d: layers(d)[1].update(container=layers(d)[2]["container"]), "layer-structure", at + "1/container"), + ("container with its own kernel", lambda d: layers(d)[2]["os"].update(kernel=fact("6.6", "synthetic:x")), "layer-structure", at + "2/os/kernel"), + ("hypervisor on a container layer", lambda d: layers(d)[2].update(technology=fact("kvm", "synthetic:x")), "layer-structure", at + "2/technology"), + ("engine on a WSL2 layer", lambda d: layers(d)[1].update(technology=fact("docker", "synthetic:x")), "layer-structure", at + "1/technology"), + ("unlisted technology on a WSL2 layer", lambda d: layers(d)[1].update(technology=fact("wsl3", "synthetic:x")), "layer-structure", at + "1/technology"), + ("WSL2 on a container layer", lambda d: layers(d)[2].update(technology=fact("wsl2", "synthetic:x")), "layer-structure", at + "2/technology"), + ("technology on the host", lambda d: layers(d)[0].update(technology=fact("hyper-v", "synthetic:x")), "layer-structure", at + "0/technology"), + ("unlisted technology on the host", lambda d: layers(d)[0].update(technology=fact("bhyve", "synthetic:x")), "layer-structure", at + "0/technology"), + ("container made by nothing", lambda d: layers(d)[2].update(technology={"value": None, "provenance": "not-applicable", "reason": "x"}), "layer-structure", at + "2/technology"), + ("WSL2 layer made by nothing", lambda d: layers(d)[1].update(technology={"value": None, "provenance": "not-applicable", "reason": "x"}), "layer-structure", at + "1/technology"), + ("WSL2 on Linux", lambda d: layers(d)[0]["os"].update(family=fact("linux", "synthetic:x")), "os-contradiction", at + "1/kind"), + ("WSL2 running Windows", lambda d: (layers(d)[1]["os"].update(family=fact("windows", "synthetic:x")), layers(d)[2]["os"].update(family=fact("windows", "synthetic:x"))), "os-contradiction", at + "1/os/family"), + ("Windows container on a Linux kernel", lambda d: layers(d)[2]["os"].update(family=fact("windows", "synthetic:x")), "os-contradiction", at + "2/os/family"), + ("emulated host", lambda d: layers(d)[0]["cpu"].update(execution=fact("emulated", "synthetic:x")), "cpu-contradiction", at + "0/cpu/execution"), + ("foreign CPU declared native", lambda d: layers(d)[2]["cpu"].update(arch=fact("x86_64", "synthetic:x")), "cpu-contradiction", at + "2/cpu/execution"), + ("architecture alias", lambda d: layers(d)[2]["cpu"].update(arch=fact("arm64", "synthetic:x")), "arch-name", at + "2/cpu/arch"), + ): + with self.subTest(label): + self.assertFinding(check(changed(result, change), policy=False), code, path) + engine_vm = changed(fixture("linux-x64-vm.result"), lambda d: layers(d)[1].update(technology=fact("podman", "synthetic:x"))) + self.assertCodes(check(engine_vm), "layer-structure") + self.assertFinding(check(engine_vm), "layer-structure", at + "1/technology") + unmade_vm = changed(fixture("linux-x64-vm.result"), lambda d: layers(d)[1].update(technology={"value": None, "provenance": "not-applicable", "reason": "x"})) + self.assertCodes(check(unmade_vm), "layer-structure") + self.assertFinding(check(unmade_vm), "layer-structure", at + "1/technology") + + def test_emulation_reaches_every_inner_layer(self): + result = fixture("vm-podman-quadlet-systemd.result") + layers = lambda d: d["environment"]["layers"] # noqa: E731 + python = lambda d: d["workload"]["runtimes"]["python"] # noqa: E731 + + def arm_python(d, execution): + python(d).update(platform=fact("linux-aarch64", "synthetic:x"), arch=fact("aarch64", "synthetic:x"), + soabi=fact("cpython-312-aarch64-linux-gnu", "synthetic:x"), execution=fact(execution, "synthetic:x")) + d["workload"]["runtimes"]["node"].update(arch=unknown(), execution=unknown()) + + def emulated_container(d): + layers(d)[2]["cpu"].update(arch=fact("aarch64", "synthetic:x"), execution=fact("emulated", "synthetic:x")) + arm_python(d, "native") + + def emulated_vm(d): + layers(d)[1]["cpu"].update(arch=fact("aarch64", "synthetic:x"), execution=fact("emulated", "synthetic:x")) + layers(d)[2]["cpu"].update(arch=fact("aarch64", "synthetic:x"), execution=fact("native", "synthetic:x")) + arm_python(d, "emulated") + + def foreign_host(d): + layers(d)[0]["cpu"].update(arch=fact("aarch64", "synthetic:x")) + layers(d)[1]["cpu"].update(arch=unknown()) + + for label, change, path in ( + ("native runtime in an emulated container", emulated_container, "/workload/runtimes/python/execution"), + ("native container in an emulated VM", emulated_vm, "/environment/layers/2/cpu/execution"), + ("x86-64 container declared native on an aarch64 host", foreign_host, "/environment/layers/2/cpu/execution"), + ): + with self.subTest(label): + self.assertFinding(check(changed(result, change)), "cpu-contradiction", path) + self.assertValid(check(changed(result, lambda d: (emulated_container(d), python(d).update(execution=fact("emulated", "synthetic:x")))))) + + def test_emulation_is_recorded_as_emulated(self): + def emulate(d): + layers = d["environment"]["layers"] + layers[2]["cpu"].update(arch=fact("x86_64", "synthetic:x"), execution=fact("emulated", "synthetic:x")) + python = d["workload"]["runtimes"]["python"] + python.update(platform=fact("linux-x86_64", "synthetic:x"), arch=fact("x86_64", "synthetic:x"), + soabi=fact("cpython-312-x86_64-linux-gnu", "synthetic:x"), execution=fact("emulated", "synthetic:x")) + d["workload"]["runtimes"]["node"].update(arch=fact("x86_64", "synthetic:x"), execution=fact("emulated", "synthetic:x")) + self.assertValid(check(changed(fixture("windows-arm64-wsl2-podman.result"), emulate))) + + def test_runtime_and_abi_contradictions(self): + result = fixture("linux-x64-vm.result") + python = lambda d: d["workload"]["runtimes"]["python"] # noqa: E731 + node = lambda d: d["workload"]["runtimes"]["node"] # noqa: E731 + for label, change, code in ( + ("Windows Python on Linux", lambda d: python(d).update(platform=fact("win-amd64", "synthetic:x")), "os-contradiction"), + ("macOS Node on Linux", lambda d: node(d).update(platform=fact("darwin", "synthetic:x")), "os-contradiction"), + ("platform tag and arch differ", lambda d: python(d).update(arch=fact("aarch64", "synthetic:x"), soabi=unknown(), execution=fact("emulated", "synthetic:x")), "abi-contradiction"), + ("32-bit pointers on x86_64", lambda d: python(d).update(pointer_bits=fact(32, "synthetic:x")), "abi-contradiction"), + ("cache tag of another version", lambda d: python(d).update(cache_tag=fact("cpython-313", "synthetic:x")), "abi-contradiction"), + ("ABI tag of another version", lambda d: python(d).update(soabi=fact("cpython-313-x86_64-linux-gnu", "synthetic:x")), "abi-contradiction"), + ("free-threaded ABI tag on a GIL build", lambda d: python(d).update(soabi=fact("cpython-312t-x86_64-linux-gnu", "synthetic:x")), "abi-contradiction"), + ("ABI tag of another architecture", lambda d: python(d).update(soabi=fact("cpython-312-aarch64-linux-gnu", "synthetic:x")), "abi-contradiction"), + ("Windows ABI tag on Linux", lambda d: python(d).update(soabi=fact("cp312-win_amd64", "synthetic:x")), "os-contradiction"), + ("no GIL on a GIL build", lambda d: python(d).update(gil_enabled=fact(False, "synthetic:x")), "abi-contradiction"), + ("glibc with a musl ABI tag", lambda d: python(d).update(soabi=fact("cpython-312-x86_64-linux-musl", "synthetic:x")), "abi-contradiction"), + ("aarch64 Node declared native on x86_64", lambda d: node(d).update(arch=fact("aarch64", "synthetic:x")), "cpu-contradiction"), + ): + with self.subTest(label): + self.assertIn(code, check(changed(result, change)).codes()) + + def test_the_abi_tag_names_the_build_architecture(self): + result = fixture("linux-x64-vm.result") + other = changed(result, lambda d: d["workload"]["runtimes"]["python"].update(soabi=fact("cpython-312-aarch64-linux-gnu", "synthetic:x"))) + self.assertCodes(check(other), "abi-contradiction") + self.assertFinding(check(other), "abi-contradiction", "/workload/runtimes/python/soabi") + # The C library in the tag must agree with the libc fact when both are known. + for libc, soabi in ((None, "cpython-312-x86_64-linux-musl"), ("glibc 2.39", "cpython-312-x86_64-linux-gnu"), ("musl 1.2.5", "cpython-312-x86_64-linux-musl"), + ("GLIBC 2.39", "cpython-312-x86_64-linux-gnu"), ("bionic", "cpython-312-x86_64-linux-android")): + with self.subTest(libc=libc, soabi=soabi): + libc_fact = unknown() if libc is None else fact(libc, "synthetic:x") + self.assertValid(check(changed(result, lambda d: d["workload"]["runtimes"]["python"].update(libc=libc_fact, soabi=fact(soabi, "synthetic:x"))))) + for libc, soabi in (("glibc 2.39", "cpython-312-x86_64-linux-musl"), ("musl 1.2.5", "cpython-312-x86_64-linux-gnu"), + ("musl 1.2.5", "cpython-312-x86_64-linux-gnux32"), ("uclibc-ng 1.0.45", "cpython-312-x86_64-linux-gnu"), + ("glibc 2.39", "cpython-312-x86_64-linux-android"), ("musl 1.2.5", "cpython-312-x86_64-linux-android"), + ("bionic", "cpython-312-x86_64-linux-musl"), ("Ubuntu GLIBC 2.39", "cpython-312-x86_64-linux-gnu"), + ("glibcx 2.39", "cpython-312-x86_64-linux-gnu")): + with self.subTest(libc=libc, soabi=soabi): + mixed = changed(result, lambda d: d["workload"]["runtimes"]["python"].update(libc=fact(libc, "synthetic:x"), soabi=fact(soabi, "synthetic:x"))) + self.assertCodes(check(mixed), "abi-contradiction") + self.assertFinding(check(mixed), "abi-contradiction", "/workload/runtimes/python/libc") + if libc in ("Ubuntu GLIBC 2.39", "glibcx 2.39"): + # The message says why a fact that mentions glibc later on, or with a letter after it, does not name it. + message = [message for _, code, _, message in check(mixed).errors if code == "abi-contradiction"][0] + self.assertIn("other than glibc or musl", message) + self.assertIn("starts with that name, in any ASCII letter case, with no letter after it", message) + # The platform part of the tag is checked even when the version is not known. + no_version = changed(result, lambda d: d["workload"]["runtimes"]["python"].update(version=unknown())) + for soabi, code in (("cpython-312-aarch64-linux-gnu", "abi-contradiction"), ("cp312-win_amd64", "os-contradiction")): + with self.subTest(soabi=soabi, version="unknown"): + self.assertFinding(check(changed(no_version, lambda d: d["workload"]["runtimes"]["python"].update(soabi=fact(soabi, "synthetic:x")))), code, "/workload/runtimes/python/soabi") + self.assertEqual(validator.soabi_target("cpython-312-arm-linux-gnueabihf"), ("linux", "armv7")) + self.assertEqual(validator.soabi_target("cpython-313t-powerpc64le-linux-gnu"), ("linux", "ppc64le")) + self.assertEqual(validator.soabi_target("cp313t-win_arm64"), ("windows", "aarch64")) + self.assertEqual(validator.soabi_target("cpython-312-darwin"), ("macos", None)) + self.assertEqual(validator.soabi_target("pypy310-pp73-x86_64-linux-gnu"), (None, None)) + self.assertEqual(validator.soabi_target("cpython-313td-x86_64-linux-gnu"), ("linux", "x86_64")) + self.assertEqual(validator.soabi_target("cpython-312d-darwin"), ("macos", None)) + self.assertEqual(validator.soabi_target("cp313d-win_amd64"), ("windows", "x86_64")) + tag = changed(result, lambda d: d["workload"]["runtimes"]["python"].update(arch=unknown(), soabi=fact("cpython-312-x86_64-linux-gnu", "synthetic:x"))) + platform_only = changed(tag, lambda d: d["workload"]["runtimes"]["python"].update(arch=fact("aarch64", "synthetic:x"), soabi=unknown(), execution=fact("emulated", "synthetic:x"))) + self.assertFinding(check(platform_only), "abi-contradiction", "/workload/runtimes/python/arch") + self.assertCodes(check(platform_only), "abi-contradiction") + # An ABI tag architecture outside the list contradicts a listed arch fact; an unknown or unlisted arch fact is not compared with it. + vax = changed(result, lambda d: d["workload"]["runtimes"]["python"].update(soabi=fact("cpython-312-vax-linux-gnu", "synthetic:x"))) + self.assertCodes(check(vax), "abi-contradiction") + self.assertFinding(check(vax), "abi-contradiction", "/workload/runtimes/python/soabi") + self.assertValid(check(changed(vax, lambda d: d["workload"]["runtimes"]["python"].update(arch=unknown(), execution=unknown())))) + for soabi in ("cpython-312-x86_64-linux-gnu", "cpython-312-vax-linux-gnu"): + with self.subTest(soabi=soabi, arch="sparc64"): + self.assertValid(check(changed(result, lambda d: d["workload"]["runtimes"]["python"].update(arch=fact("sparc64", "synthetic:x"), soabi=fact(soabi, "synthetic:x"))))) + _, real = real_result() + self.assertFinding(check(changed(real, lambda d: d["workload"]["runtimes"]["python"].update(soabi=fact("cpython-312-vax-linux-gnu", provenance="declared")))), + "abi-contradiction", "/workload/runtimes/python/soabi") + + def test_a_cpython_version_is_read_whole(self): + result = fixture("linux-x64-vm.result") + _, real = real_result() + python = lambda d: d["workload"]["runtimes"]["python"] # noqa: E731 + for version in ("3.12", "3.12.0", "3.12.10", "3.12.15", "3.12.15+", "3.12.0a0", "3.12.0a1", "3.12.0b2", "3.12.0rc1", "3.12.0a1+"): + with self.subTest(version=version): + self.assertEqual(validator.cpython_version(version), ("3", "12")) + self.assertValid(check(changed(result, lambda d: python(d).update(version=fact(version, "synthetic:x"))))) + # Each is CPython 3.12, the lock's interpreter; the lane's checked status needs 3.12.13 or a later release. + claimed = check(changed(real, lambda d: python(d).update(version=fact(version, provenance="declared")))) + expected = [] if version == "3.12.15" else [("support-claim", "/workload/runtimes/python/version")] + self.assertEqual([(code, path) for _, code, path, _ in claimed.errors], expected) + # Text after the version is not ignored: the version is an error, and the profile loses its lane. + for version in ("3.12-not-python", "3.12.15garbage", "3.12x", "3.12+", "3.12rc1", "3.12.15 ", " 3.12.15", "3.12.15rc", "3.12.\u0661\u0665", "3.12.1\u0665", "3.12.015", "03.12.15", "3.012", "3.12.0a01", "3.12.0rc01"): + with self.subTest(version=version): + self.assertIsNone(validator.cpython_version(version)) + self.assertCodes(check(changed(result, lambda d: python(d).update(version=fact(version, "synthetic:x")))), "abi-contradiction") + self.assertFinding(check(changed(result, lambda d: python(d).update(version=fact(version, "synthetic:x")))), "abi-contradiction", "/workload/runtimes/python/version") + claimed = check(changed(real, lambda d: python(d).update(version=fact(version, provenance="declared")))) + self.assertFinding(claimed, "abi-contradiction", "/workload/runtimes/python/version") + self.assertFinding(claimed, "support-claim", "/workload/runtimes/python") + self.assertIsNone(validator.cpython_version("3.12.15\n")) + message = [message for _, code, _, message in check(changed(real, lambda d: python(d).update(version=fact("3.12-not-python", provenance="declared")))).errors if code == "support-claim"][0] + self.assertIn("declares cpython '3.12-not-python'", message) + message = [message for _, code, _, message in check(changed(real, lambda d: python(d).update(version=fact("3.13.0", provenance="declared")))).errors if code == "support-claim"][0] + self.assertIn("declares cpython 3.13 (free-threaded build", message) + # The rule is for CPython, whose version the cache tag and the ABI tag repeat. + pypy = changed(result, lambda d: python(d).update(implementation=fact("pypy", "synthetic:x"), version=fact("3.10.14-pypy", "synthetic:x"), cache_tag=unknown(), soabi=unknown())) + self.assertNotIn("/workload/runtimes/python/version", [path for _, _, path, _ in check(pypy).items]) + + def test_an_abi_tag_the_validator_cannot_read_is_reported(self): + result = fixture("linux-x64-vm.result") + odd = changed(result, lambda d: d["workload"]["runtimes"]["python"].update(soabi=fact("cpython-312-x86_64-cygwin", "synthetic:x"))) + findings = check(odd) + self.assertValid(findings) + self.assertFinding(findings, "unrecognized-value", "/workload/runtimes/python/soabi", level="warning") + debug = changed(result, lambda d: d["workload"]["runtimes"]["python"].update(soabi=fact("cpython-312d-x86_64-linux-gnu", "synthetic:x"))) + self.assertNotIn("unrecognized-value", check(debug).codes("warning")) + + def test_windows_and_macos_platform_tags_name_the_build(self): + result = fixture("windows-arm64-native.result") + python = lambda d: d["workload"]["runtimes"]["python"] # noqa: E731 + x86_on_arm = changed(result, lambda d: python(d).update(platform=fact("win32", "synthetic:x"), arch=fact("x86_64", "synthetic:x"), + soabi=unknown(), execution=fact("emulated", "synthetic:x"))) + self.assertFinding(check(x86_on_arm), "abi-contradiction", "/workload/runtimes/python/arch") + amd64_tag = changed(result, lambda d: python(d).update(soabi=fact("cp312-win_amd64", "synthetic:x"))) + self.assertFinding(check(amd64_tag), "abi-contradiction", "/workload/runtimes/python/soabi") + self.assertEqual(validator.python_platform_target("macosx-14.0-universal2"), ("macos", None)) + # glibc and musl are C libraries for Linux, so they contradict a Windows or macOS Python whatever its ABI tag names. + for document in (result, fixture("windows-arm64-native.profile")): + for libc in ("glibc 2.39", "GLIBC 2.39", "musl 1.2.5"): + with self.subTest(document=document["kind"], libc=libc): + self.assertFinding(check(changed(document, lambda d: python(d).update(libc=fact(libc, provenance="declared")))), "abi-contradiction", "/workload/runtimes/python/libc") + self.assertValid(check(changed(result, lambda d: python(d).update(libc=fact("ucrt", "synthetic:x"))))) + + # Each of the innermost layer, the platform tag and the ABI tag is enough on its own. + def layer(family): + def change(d): + d["environment"]["layers"][-1]["os"]["family"] = fact(family, "synthetic:x") + python(d).update(platform=unknown(), soabi=unknown()) + return change + + for source, change in ( + ("windows layer", layer("windows")), + ("macos layer", layer("macos")), + ("freebsd layer", layer("freebsd")), + ("windows platform", lambda d: python(d).update(platform=fact("win-amd64", "synthetic:x"), soabi=unknown())), + ("macos platform", lambda d: python(d).update(platform=fact("macosx-14.0-x86_64", "synthetic:x"), soabi=unknown())), + ("freebsd platform", lambda d: python(d).update(platform=fact("freebsd-14.1-RELEASE-amd64", "synthetic:x"), soabi=unknown())), + ("windows soabi", lambda d: python(d).update(platform=unknown(), soabi=fact("cp312-win_amd64", "synthetic:x"))), + ("macos soabi", lambda d: python(d).update(platform=unknown(), soabi=fact("cpython-312-darwin", "synthetic:x"))), + ("both tags", lambda d: python(d).update(platform=fact("macosx-14.0-arm64", "synthetic:x"), soabi=fact("cpython-312-darwin", "synthetic:x"))), + ): + with self.subTest(source=source): + self.assertFinding(check(changed(fixture("linux-x64-vm.result"), change)), "abi-contradiction", "/workload/runtimes/python/libc") + + def test_the_platform_tag_and_the_abi_tag_are_compared_with_each_other(self): + # Each CPython tag is compared with the innermost layer and the arch fact; when one of those is unknown, with the other tag. + python = lambda d: d["workload"]["runtimes"]["python"] # noqa: E731 + + def tags(document, platform, soabi, family=None, arch=None): + def change(d): + if family is not None: + d["environment"]["layers"][-1]["os"]["family"] = family + if arch is not None: + python(d).update(arch=arch, pointer_bits=unknown()) + python(d).update(platform=fact(platform, "synthetic:x"), soabi=fact(soabi, "synthetic:x"), libc=unknown()) + return changed(fixture(document), change) + + for platform, soabi in (("win-amd64", "cpython-312-x86_64-linux-gnu"), ("linux-x86_64", "cp312-win_amd64"), ("macosx-14.0-x86_64", "cp312-win_amd64")): + with self.subTest(platform=platform, soabi=soabi): + findings = check(tags("linux-x64-vm.result", platform, soabi, family=unknown()), policy=False) + self.assertCodes(findings, "os-contradiction") + self.assertFinding(findings, "os-contradiction", "/workload/runtimes/python/soabi") + for document, platform, soabi, arch in ( + ("linux-x64-vm.result", "linux-aarch64", "cpython-312-x86_64-linux-gnu", unknown()), + ("linux-x64-vm.result", "linux-aarch64", "cpython-312-x86_64-linux-gnu", fact("vax", "synthetic:x")), + ("windows-arm64-native.result", "win-amd64", "cp312-win_arm64", unknown()), + ): + with self.subTest(platform=platform, soabi=soabi, arch=arch["value"]): + findings = check(tags(document, platform, soabi, arch=arch)) + self.assertCodes(findings, "abi-contradiction") + self.assertFinding(findings, "abi-contradiction", "/workload/runtimes/python/soabi") + # A 32-bit build on a 64-bit Linux kernel agrees with its platform tag, and tags that agree pass without the other facts. + self.assertValid(check(tags("linux-x64-vm.result", "linux-x86_64", "cpython-312-i386-linux-gnu", arch=unknown()))) + self.assertValid(check(tags("linux-x64-vm.result", "linux-x86_64", "cpython-312-x86_64-linux-gnu", family=unknown(), arch=unknown()), policy=False)) + # When the layer is known, a tag that disagrees is reported against it, once, and tags of two OSes are not compared by architecture. + for arch in (None, unknown()): + with self.subTest(arch=arch): + findings = check(tags("linux-x64-vm.result", "win32", "cpython-312-x86_64-linux-gnu", arch=arch)) + self.assertEqual([(code, path) for _, code, path, _ in findings.errors], [("os-contradiction", "/workload/runtimes/python/platform")] if arch else + [("os-contradiction", "/workload/runtimes/python/platform"), ("abi-contradiction", "/workload/runtimes/python/arch")]) + # As with the layer and the arch fact, the ABI tag is read for CPython only. + other = changed(tags("linux-x64-vm.result", "win-amd64", "cpython-312-x86_64-linux-gnu", family=unknown()), lambda d: python(d).update(implementation=unknown())) + self.assertNotIn("/workload/runtimes/python/soabi", [path for _, _, path, _ in check(other, policy=False).errors]) + + def test_32_bit_code_on_a_64_bit_kernel_of_its_family_is_native(self): + def thirty_two_bit(d): + d["workload"]["runtimes"]["python"].update(arch=fact("x86", "synthetic:x"), pointer_bits=fact(32, "synthetic:x"), + soabi=fact("cpython-312-i386-linux-gnu", "synthetic:x")) + self.assertValid(check(changed(fixture("linux-x64-vm.result"), thirty_two_bit))) + + def x86_on_arm(d): + d["workload"]["runtimes"]["python"].update(arch=fact("x86", "synthetic:x"), pointer_bits=fact(32, "synthetic:x")) + self.assertIn("cpu-contradiction", check(changed(fixture("windows-arm64-wsl2-podman.result"), x86_on_arm)).codes()) + + def test_a_free_threaded_build_is_consistent(self): + def free_threaded(d): + python = d["workload"]["runtimes"]["python"] + python.update(version=fact("3.15.0", "synthetic:x"), cache_tag=fact("cpython-315", "synthetic:x"), + soabi=fact("cpython-315t-x86_64-linux-gnu", "synthetic:x"), + free_threaded_build=fact(True, "synthetic:x"), gil_enabled=fact(False, "synthetic:x")) + self.assertValid(check(changed(fixture("linux-x64-vm.result"), free_threaded))) + + def test_unrecognized_values_warn_and_change_nothing_else(self): + def unusual(d): + d["profile_id"] = None + layer = d["environment"]["layers"][1] + layer["cpu"]["arch"] = fact("e2k", "synthetic:x") + layer["technology"] = fact("bhyve", "synthetic:x") + layer["os"]["family"] = fact("haiku", "synthetic:x") + d["workload"]["runtimes"]["python"].update(platform=fact("haiku-e2k", "synthetic:x"), arch=fact("e2k", "synthetic:x"), + soabi=unknown(), execution=unknown()) + d["workload"]["runtimes"]["node"].update(platform=fact("haiku", "synthetic:x"), arch=unknown(), execution=unknown()) + d["compute"]["backends"].append({"id": "npu0", "kind": "npu", "api": "vendor-sdk", "available": unknown(), + "device": unknown(), "version": unknown(), "features": unknown()}) + findings = check(changed(fixture("linux-x64-vm.result"), unusual)) + self.assertValid(findings) + self.assertCodes(findings, "unrecognized-value", level="warning") + for path in ("/environment/layers/1/cpu/arch", "/environment/layers/1/os/family", "/environment/layers/1/technology", "/compute/backends/1/api"): + self.assertFinding(findings, "unrecognized-value", path, level="warning") + + def test_identity(self): + result = fixture("windows-arm64-wsl2-podman.result") + identity = lambda d: d["workload"]["identity"] # noqa: E731 + superuser = changed(result, lambda d: identity(d).update(superuser=fact(True, "synthetic:x"))) + self.assertIn("identity-contradiction", check(superuser, policy=False).codes()) + initial = changed(result, lambda d: identity(d)["user_namespace"].update(initial=fact(True, "synthetic:x"))) + self.assertCodes(check(initial), "identity-contradiction") + for family in ("macos", "freebsd"): + with self.subTest(family): + root = changed(fixture("linux-x64-vm.result"), lambda d, family=family: ( + d["environment"]["layers"][-1]["os"].update(family=fact(family, "synthetic:x")), identity(d).update(uid=fact(0, "synthetic:x")))) + self.assertIn("identity-contradiction", check(root, policy=False).codes()) + nested_full_map = changed(fixture("linux-x64-vm.result"), lambda d: identity(d)["user_namespace"].update(initial=fact(False, "synthetic:x"))) + self.assertValid(check(nested_full_map)) + + +class StageRuleTests(Case): + def stage(self, document, name, **values): + def change(d): + d["stages"][name] = values + d["overall"] = validator.overall_of(d["stages"]) + return changed(document, change) + + def test_stage_fields(self): + result = fixture("linux-x64-vm.result") + escalation = {"escalation": {"value": "none", "provenance": "declared"}} + for label, values in ( + ("not-run with evidence", dict(status="not-run", reason="x", evidence=["synthetic:log"])), + ("not-run without reason", dict(status="not-run")), + ("failed without evidence", dict(status="failed", reason="x", privilege=escalation)), + ("failed without reason", dict(status="failed", evidence=["synthetic:log"], privilege=escalation)), + ("passed without evidence", dict(status="passed", privilege=escalation)), + ("passed without privilege", dict(status="passed", evidence=["synthetic:log"])), + ("blocked without reason", dict(status="blocked")), + ): + with self.subTest(label): + self.assertIn("stage-fields", check(self.stage(result, "install", **values)).codes()) + + def test_only_security_and_performance_can_be_not_applicable(self): + result = fixture("linux-x64-vm.result") + for name in ("detect", "install", "initialize", "execute", "correctness"): + with self.subTest(name): + self.assertFinding(check(self.stage(result, name, status="not-applicable", reason="x")), "stage-fields", f"/stages/{name}/status") + for name in ("security", "performance"): + with self.subTest(name): + self.assertValid(check(self.stage(result, name, status="not-applicable", reason="x"))) + + # Every stage after detect marked not-applicable does not make a passed run. + def detect_only(d): + for name in validator.STAGES[1:]: + d["stages"][name] = {"status": "not-applicable", "reason": "x"} + d["overall"] = validator.overall_of(d["stages"]) + findings = check(changed(result, detect_only)) + self.assertCodes(findings, "stage-fields") + self.assertFinding(findings, "stage-fields", "/stages/install/status") + + def test_a_real_stage_cites_a_log_or_an_artifact(self): + result, profile = real_result() + passed = dict(status="passed", privilege={"escalation": {"value": "none", "provenance": "declared"}}) + for name, evidence, valid in ( + ("install", ["doc:README.md"], False), + ("install", ["tool:collect-environment"], False), + ("install", ["file:/proc/self/status"], False), + ("install", ["log:uv pip sync"], True), + ("install", ["artifact:install-log"], True), + ("detect", ["doc:README.md"], False), + ("detect", ["tool:collect-environment"], True), + ): + with self.subTest(name=name, evidence=evidence): + findings = check(self.stage(result, name, evidence=evidence, **passed), profile) + if valid: + self.assertValid(findings) + else: + self.assertFinding(findings, "stage-fields", f"/stages/{name}/evidence") + + def test_detect_cannot_pass_with_probe_errors(self): + result = fixture("linux-x64-vm.result") + self.assertEqual(result["stages"]["detect"]["status"], "passed") + broken = changed(result, lambda d: d.update(probe_errors=[{"probe": "isolation", "error": "ValueError raised by the isolation probe"}])) + self.assertCodes(check(broken), "stage-fields") + self.assertFinding(check(broken), "stage-fields", "/stages/detect/status") + + def test_a_stage_cannot_pass_before_its_prerequisites(self): + result = fixture("linux-x64-vm.result") + passed = dict(status="passed", evidence=["synthetic:log"], privilege={"escalation": {"value": "none", "provenance": "declared"}}) + for name in ("initialize", "correctness", "performance"): + with self.subTest(name): + self.assertCodes(check(self.stage(result, name, **passed)), "stage-order") + ran = result + for name in ("detect", "install", "initialize", "execute"): + ran = self.stage(ran, name, **passed) + ran = self.stage(ran, "correctness", status="failed", reason="wrong output", evidence=["synthetic:log"], privilege=passed["privilege"]) + self.assertFinding(check(self.stage(ran, "performance", **passed)), "stage-order", "/stages/performance/status") + failed_detect = self.stage(result, "detect", status="failed", reason="probe error", evidence=["synthetic:log"], privilege=passed["privilege"]) + self.assertCodes(check(self.stage(failed_detect, "security", **passed)), "stage-order") + + def test_not_run_stages_cannot_make_the_run_pass(self): + result = fixture("linux-x64-vm.result") + self.assertEqual(result["overall"], "incomplete") + self.assertCodes(check(changed(result, lambda d: d.update(overall="passed"))), "overall-mismatch") + failed = fixture("windows-arm64-native.result") + self.assertEqual(failed["overall"], "failed") + self.assertCodes(check(changed(failed, lambda d: d.update(overall="incomplete"))), "overall-mismatch") + + def test_a_blocked_stage_leaves_the_run_incomplete(self): + result = fixture("linux-x64-vm.result") + passed = dict(status="passed", evidence=["synthetic:log"], privilege={"escalation": {"value": "none", "provenance": "declared"}}) + ran = result + for name in ("install", "initialize", "execute", "correctness"): + ran = self.stage(ran, name, **passed) + ran = self.stage(ran, "security", status="not-applicable", reason="x") + ran = self.stage(ran, "performance", status="blocked", reason="no benchmark runner") + self.assertEqual(validator.overall_of(ran["stages"]), "incomplete") + self.assertValid(check(ran)) + for overall in ("passed", "failed"): + with self.subTest(overall=overall): + self.assertFinding(check(changed(ran, lambda d: d.update(overall=overall))), "overall-mismatch", "/overall") + failed = self.stage(ran, "correctness", status="failed", reason="wrong output", evidence=["synthetic:log"], privilege=passed["privilege"]) + self.assertEqual(validator.overall_of(failed["stages"]), "failed") + + def test_a_failure_is_not_recorded_as_not_run(self): + result = fixture("windows-arm64-native.result") + hidden = self.stage(result, "install", status="not-run", reason="x", evidence=["synthetic:uv pip sync log"]) + self.assertCodes(check(hidden), "stage-fields") + + def test_required_stages_must_pass(self): + result, profile = pair("windows-arm64-wsl2-podman") + skipped = self.stage(result, "correctness", status="not-run", reason="x") + self.assertCodes(check(skipped, profile), "required-stage") + + def test_cpu_fallback_is_not_an_accelerator_pass(self): + result, profile = pair("kubernetes-arm64-cuda") + execute = lambda d: d["stages"]["execute"] # noqa: E731 + for label, change, code, path in ( + ("fell back to the CPU", lambda d: execute(d)["compute"].update(effective="cpu"), "accelerator-fallback", "/stages/execute/compute"), + ("nothing recorded as running", lambda d: execute(d)["compute"].update(effective=None), "accelerator-fallback", "/stages/execute/compute"), + ("CPU requested, nothing recorded as running", lambda d: execute(d)["compute"].update(requested="cpu", effective=None), "backend-reference", "/stages/execute/compute/effective"), + ("unknown backend requested", lambda d: execute(d)["compute"].update(requested="rocm"), "backend-reference", "/stages/execute/compute/requested"), + ("unknown backend ran", lambda d: execute(d)["compute"].update(effective="rocm"), "backend-reference", "/stages/execute/compute/effective"), + ("unavailable backend ran", lambda d: d["compute"]["backends"][1].update(available=unknown()), "backend-reference", "/stages/execute/compute/effective"), + ("duplicate backend id", lambda d: d["compute"]["backends"].append(copy.deepcopy(d["compute"]["backends"][1])), "backend-reference", "/compute/backends/1/id"), + ("CPU API on a GPU", lambda d: d["compute"]["backends"][1].update(api="cpu"), "backend-reference", "/compute/backends/1"), + ): + with self.subTest(label): + self.assertFinding(check(changed(result, change), profile), code, path) + other_kind = {"id": "fpga", "kind": "other", "api": "opencl", "available": fact(True, "synthetic:x"), "device": unknown(), "version": unknown(), "features": unknown()} + for label, change in ( + ("another kind of backend requested", lambda d: (d["compute"]["backends"].append(other_kind), execute(d)["compute"].update(requested="fpga", effective=None))), + ("an unlisted backend requested", lambda d: execute(d)["compute"].update(requested="rocm", effective=None)), + ): + with self.subTest(label): + self.assertFinding(check(changed(result, change), profile), "backend-reference", "/stages/execute/compute/effective") + # A stage that failed may have failed before any backend ran. + failed = check(changed(result, lambda d: (execute(d).update(status="failed", reason="x"), execute(d)["compute"].update(requested="cpu", effective=None))), profile) + self.assertNotIn(("backend-reference", "/stages/execute/compute/effective"), [(code, path) for _, code, path, _ in failed.errors]) + # A GPU stage that records no backend is reported once, as a fallback. + nothing = check(changed(result, lambda d: execute(d)["compute"].update(effective=None)), profile) + self.assertNotIn(("backend-reference", "/stages/execute/compute/effective"), [(code, path) for _, code, path, _ in nothing.errors]) + other_gpu = {"id": "cuda1", "kind": "gpu", "api": "cuda", "available": fact(True, "synthetic:x"), "device": unknown(), "version": unknown(), "features": unknown()} + to_other_gpu = changed(result, lambda d: (d["compute"]["backends"].append(other_gpu), execute(d)["compute"].update(effective="cuda1"))) + self.assertFinding(check(to_other_gpu), "accelerator-fallback", "/stages/execute/compute") + npu = fixture("windows-arm64-native.result") + npu_id = next(backend["id"] for backend in npu["compute"]["backends"] if backend["kind"] == "npu") + + def npu_on_cpu(d): + for name in ("detect", "install", "initialize", "execute"): + d["stages"][name] = {"status": "passed", "evidence": ["synthetic:log"], "privilege": {"escalation": {"value": "none", "provenance": "declared"}}} + d["stages"]["execute"]["compute"] = {"requested": npu_id, "effective": "cpu"} + d["overall"] = validator.overall_of(d["stages"]) + self.assertFinding(check(changed(npu, npu_on_cpu)), "accelerator-fallback", "/stages/execute/compute") + + def test_an_accelerator_profile_needs_its_stages_on_the_accelerator(self): + result, profile = pair("kubernetes-arm64-cuda") + self.assertEqual(profile["accelerator_backend"], "cuda") + + def on_cpu(d): + d["compute"]["backends"][1]["available"] = fact(False, "synthetic:x") + for name in ("execute", "correctness"): + d["stages"][name]["compute"] = {"requested": "cpu", "effective": "cpu"} + + def unrecorded(d): + for name in ("execute", "correctness"): + d["stages"][name].pop("compute") + + for label, change in (("ran on the CPU", on_cpu), ("no backend recorded", unrecorded)): + with self.subTest(label): + findings = check(changed(result, change), profile) + self.assertFinding(findings, "required-stage", "/stages/execute/compute") + self.assertFinding(findings, "required-stage", "/stages/correctness/compute") + # The CPU was asked for, so this is no fallback, but the stage still did not request the accelerator. + cpu_requested = changed(result, lambda d: d["stages"]["execute"]["compute"].update(requested="cpu")) + self.assertCodes(check(cpu_requested, profile), "required-stage") + self.assertFinding(check(cpu_requested, profile), "required-stage", "/stages/execute/compute") + + def performance_on_cpu(d): + d["stages"]["performance"] = {"status": "passed", "evidence": ["synthetic:benchmark log"], "privilege": {"escalation": {"value": "none", "provenance": "declared"}}, + "compute": {"requested": "cpu", "effective": "cpu"}} + performance_profile = changed(profile, lambda d: d["required_stages"].append("performance")) + self.assertFinding(check(changed(result, performance_on_cpu), performance_profile), "required-stage", "/stages/performance/compute") + for label, name in (("missing backend", "rocm"), ("CPU backend", "cpu")): + with self.subTest(label): + self.assertFinding(check(changed(profile, lambda d: d.update(accelerator_backend=name))), "backend-reference", "/accelerator_backend") + # A CPU lane names no accelerator backend. + self.assertValid(check(changed(profile, lambda d: d.update(accelerator_backend=None)))) + + def test_a_cpu_backend_may_use_another_api(self): + result = fixture("linux-x64-vm.result") + wasm = {"id": "wasm", "kind": "cpu", "api": "wasm", "available": unknown(), "device": unknown(), "version": unknown(), "features": unknown()} + self.assertValid(check(changed(result, lambda d: d["compute"]["backends"].append(wasm)))) + + def test_a_failed_stage_may_record_the_fallback_it_hit(self): + result, profile = pair("kubernetes-arm64-cuda") + + def fail(d): + d["stages"]["correctness"].update(status="failed", reason="the CUDA kernel did not load; the CPU ran instead") + d["stages"]["correctness"]["compute"]["effective"] = "cpu" + d["overall"] = "failed" + findings = check(changed(result, fail), profile) + self.assertCodes(findings, "required-stage") + + def test_a_stage_that_did_not_run_used_no_backend(self): + result = fixture("linux-x64-vm.result") + document = changed(result, lambda d: d["stages"]["install"].update(compute={"requested": "cpu", "effective": "cpu"})) + self.assertCodes(check(document), "backend-reference") + + +class PolicyTests(Case): + def test_the_default_policy_rejects_root_privileged_devices_capabilities_and_escalation(self): + result = fixture("windows-arm64-wsl2-podman.result") + container = lambda d: d["environment"]["layers"][2]["container"] # noqa: E731 + for label, change, code in ( + ("root workload", lambda d: d["workload"]["identity"].update(superuser=fact(True, "synthetic:x"), uid=fact(0, "synthetic:x")), "policy-app-non-root"), + ("unknown user", lambda d: d["workload"]["identity"].update(superuser=unknown()), "policy-app-non-root"), + ("privileged container", lambda d: container(d).update(privileged=fact(True, "synthetic:x")), "policy-no-privileged-container"), + ("unknown privileged state", lambda d: container(d).update(privileged=unknown()), "policy-no-privileged-container"), + ("device passed in", lambda d: container(d).update(devices=fact(["fuse"], "synthetic:x")), "policy-minimal-devices"), + ("capability held", lambda d: d["workload"]["isolation"]["capabilities"].update(effective=fact(["cap_net_admin"], "synthetic:x")), "policy-minimal-capabilities"), + ("sudo in a stage", lambda d: d["stages"]["install"]["privilege"].update(escalation={"value": "sudo", "provenance": "declared"}), "policy-no-privilege-escalation"), + ): + with self.subTest(label): + self.assertCodes(check(changed(result, change)), code) + + def test_capabilities_apply_on_linux(self): + result = fixture("windows-arm64-wsl2-podman.result") + not_applicable = {"value": None, "provenance": "not-applicable", "reason": "x"} + findings = check(changed(result, lambda d: d["workload"]["isolation"]["capabilities"].update(effective=not_applicable))) + self.assertFinding(findings, "policy-minimal-capabilities", "/workload/isolation/capabilities/effective") + # Windows has no Linux capabilities. + self.assertValid(check(fixture("windows-arm64-native.result"))) + # An unknown OS family is not shown to be one without them. + unknown_family = changed(result, lambda d: (d["environment"]["layers"][-1]["os"].update(family=unknown()), + d["workload"]["isolation"]["capabilities"].update(effective=not_applicable))) + self.assertFinding(check(unknown_family), "policy-minimal-capabilities", "/workload/isolation/capabilities/effective") + + def test_a_workload_not_shown_outside_a_container_does_not_pass_the_container_rules(self): + result = fixture("linux-x64-vm.result") + path = "/workload/isolation/pid_namespace/initial" + for label, initial in (("another PID namespace", fact(False, "synthetic:x")), ("unknown", unknown())): + with self.subTest(label): + findings = check(changed(result, lambda d: d["workload"]["isolation"]["pid_namespace"].update(initial=initial))) + self.assertCodes(findings, "policy-minimal-devices", "policy-no-privileged-container") + self.assertFinding(findings, "policy-no-privileged-container", path) + self.assertFinding(findings, "policy-engine-rootless", path, level="warning") + self.assertValid(check(result)) + # An unknown OS family fails closed, like Linux. + unknown_family = changed(result, lambda d: (d["environment"]["layers"][-1]["os"].update(family=unknown()), + d["workload"]["isolation"]["pid_namespace"].update(initial=fact(False, "synthetic:x")))) + self.assertFinding(check(unknown_family), "policy-no-privileged-container", path) + # WSL 2 gives every distribution its own PID namespace, so there the namespace does not suggest a container. + wsl2 = fixture("windows-arm64-wsl2-podman.result") + distribution = changed(wsl2, lambda d: (d["environment"]["layers"].pop(2), d.update(profile_id=None), + d["workload"]["isolation"]["pid_namespace"].update(initial=fact(False, "synthetic:x")))) + self.assertValid(check(distribution)) + self.assertCodes(check(distribution), level="warning") + + def test_rootless_engine_and_non_root_workload_are_independent(self): + result, profile = pair("windows-arm64-wsl2-podman") + root_in_rootless = changed(result, lambda d: d["workload"]["identity"].update( + superuser=fact(True, "synthetic:x"), uid=fact(0, "synthetic:x"), + user_namespace={"initial": fact(False, "synthetic:x"), "uid_map": fact([[0, 1000, 1], [1, 100000, 65536]], "synthetic:x")})) + self.assertCodes(check(root_in_rootless, profile), "policy-app-non-root") + rootful = changed(result, lambda d: (d["environment"]["layers"][2]["container"].update(engine_mode=fact("rootful", "synthetic:x")), d.update(profile_id=None))) + findings = check(rootful) + self.assertCodes(findings) + self.assertCodes(findings, "policy-engine-rootless", level="warning") + rootful_profile = changed(profile, lambda d: d["environment"]["layers"][2]["container"].update(engine_mode=unknown())) + self.assertCodes(check(changed(rootful, lambda d: d.update(profile_id=profile["id"])), rootful_profile), "policy-engine-rootless") + + def test_exceptions_need_approval_and_a_matching_scope(self): + result, profile = pair("kubernetes-arm64-cuda") + exception = lambda d: d["policy"]["exceptions"][0] # noqa: E731 + pending = changed(profile, lambda d: exception(d)["approval"].update(status="pending")) + self.assertCodes(check(result, pending), "policy-exception-not-approved") + other_device = changed(profile, lambda d: exception(d)["scope"].update(devices=["fuse"])) + self.assertCodes(check(result, other_device), "policy-minimal-devices") + self.assertCodes(check(result), "policy-minimal-devices") + for label, change in ( + ("approved without a reference", lambda d: exception(d)["approval"].pop("reference")), + ("scope of another rule", lambda d: exception(d)["scope"].update(stages=["execute"])), + ("no scope", lambda d: exception(d).update(scope={})), + ("duplicate id", lambda d: d["policy"]["exceptions"].append(copy.deepcopy(exception(d)))), + ): + with self.subTest(label): + self.assertIn("policy-exception", check(changed(profile, change)).codes()) + # The schema checks only the date's shape; the day must exist. + for date in ("2026-99-99", "2026-02-30", "2026-00-10", "2025-02-29"): + with self.subTest(date=date): + self.assertFinding(check(changed(profile, lambda d: exception(d)["approval"].update(date=date))), "policy-exception", "/policy/exceptions/0/approval/date") + self.assertValid(check(changed(profile, lambda d: exception(d)["approval"].update(date="2024-02-29")))) + for label, change in ( + ("no reason", lambda d: exception(d).pop("reason")), + ("no end condition", lambda d: exception(d).pop("end_condition")), + ("no verification", lambda d: exception(d).update(verification="")), + ("unknown rule", lambda d: exception(d).update(rule="no-network")), + ): + with self.subTest(label): + self.assertCodes(check(changed(profile, change)), "schema") + + def test_unused_exceptions_warn(self): + result, profile = pair("kubernetes-arm64-cuda") + no_gpu = changed(result, lambda d: d["environment"]["layers"][1]["container"].update(devices=fact([], "synthetic:x"))) + self.assertIn("policy-exception-unused", check(no_gpu, changed(profile, lambda d: d["environment"]["layers"][1]["container"].update(devices=unknown()))).codes("warning")) + + def test_a_profile_can_tighten_but_not_weaken_a_rule(self): + profile = fixture("linux-x64-vm.profile") + self.assertCodes(check(changed(profile, lambda d: d["policy"]["rules"].update({"app-non-root": "preferred"}))), "policy-weakened") + self.assertValid(check(changed(profile, lambda d: d["policy"]["rules"].update({"engine-rootless": "required"})))) + + def test_the_policy_can_be_skipped_for_machines_it_is_not_meant_for(self): + result = changed(fixture("linux-x64-vm.result"), lambda d: d["workload"]["identity"].update(superuser=fact(True, "synthetic:x"), uid=fact(0, "synthetic:x"))) + self.assertCodes(check(result), "policy-app-non-root") + self.assertValid(check(result, policy=False)) + + +class MeasuredPolicyTests(Case): + """The policy passes only on measured facts, fails closed off Linux, and lets an exception cover only what it names.""" + + def test_declared_values_in_a_result_do_not_show_compliance(self): + result = fixture("windows-arm64-wsl2-podman.result") + container = lambda d: d["environment"]["layers"][2]["container"] # noqa: E731 + identity = lambda d: d["workload"]["identity"] # noqa: E731 + stated = lambda value: fact(value, provenance="declared") # noqa: E731 + for label, change, code, path in ( + ("privileged", lambda d: container(d).update(privileged=stated(False)), "policy-no-privileged-container", "/environment/layers/2/container/privileged"), + ("devices", lambda d: container(d).update(devices=stated([])), "policy-minimal-devices", "/environment/layers/2/container/devices"), + ("superuser", lambda d: identity(d).update(superuser=stated(False)), "policy-app-non-root", "/workload/identity/superuser"), + ("real user ID", lambda d: identity(d).update(real_uid=stated(1000)), "policy-app-non-root", "/workload/identity/real_uid"), + ("effective capabilities", lambda d: d["workload"]["isolation"]["capabilities"].update(effective=stated([])), "policy-minimal-capabilities", "/workload/isolation/capabilities/effective"), + ): + with self.subTest(label): + findings = check(changed(result, change)) + self.assertCodes(findings, code) + self.assertFinding(findings, code, path) + vm = fixture("linux-x64-vm.result") + outside = changed(vm, lambda d: d["workload"]["isolation"]["pid_namespace"].update(initial=stated(True))) + self.assertFinding(check(outside), "policy-no-privileged-container", "/workload/isolation/pid_namespace/initial") + # A layer that a declared fact calls a container is still held to the container rules. + declared_kind = changed(result, lambda d: (d["environment"]["layers"][2].update(kind=stated("container")), container(d).update(privileged=unknown()))) + self.assertFinding(check(declared_kind), "policy-no-privileged-container", "/environment/layers/2/container/privileged") + + def test_unknown_or_unshown_privilege_state_fails(self): + result, profile = pair("windows-arm64-wsl2-podman") + unknown_mode = changed(result, lambda d: (d["environment"]["layers"][2]["container"].update(engine_mode=unknown()), d.update(profile_id=None))) + self.assertCodes(check(unknown_mode), level="error") + self.assertCodes(check(unknown_mode), "policy-engine-rootless", level="warning") + required = changed(profile, lambda d: (d["policy"]["rules"].update({"engine-rootless": "required"}), d["environment"]["layers"][2]["container"].update(engine_mode=unknown()))) + self.assertCodes(check(changed(unknown_mode, lambda d: d.update(profile_id=profile["id"])), required), "policy-engine-rootless") + self.assertCodes(check(changed(result, lambda d: d["stages"]["install"]["privilege"].update(escalation=unknown()))), "policy-no-privilege-escalation") + failed_with_sudo = changed(result, lambda d: (d["stages"]["correctness"].update(status="failed", reason="x"), + d["stages"]["correctness"]["privilege"].update(escalation=fact("sudo", provenance="declared")), + d.update(overall="failed"))) + self.assertCodes(check(failed_with_sudo), "policy-no-privilege-escalation") + + def test_off_linux_an_unmeasured_layer_kind_fails_closed(self): + windows = fixture("windows-arm64-native.result") + self.assertValid(check(windows)) + layer = lambda d: d["environment"]["layers"][0] # noqa: E731 + path = "/environment/layers/0/kind" + for label, change in ( + ("Windows, kind unknown", lambda d: layer(d).update(kind=unknown())), + ("Windows, kind declared", lambda d: layer(d).update(kind=fact("host", provenance="declared"))), + ("FreeBSD, kind unknown", lambda d: (layer(d).update(kind=unknown()), layer(d)["os"].update(family=fact("freebsd", "synthetic:x")))), + ): + with self.subTest(label): + findings = check(changed(windows, change), policy=True) + self.assertFinding(findings, "policy-no-privileged-container", path) + self.assertFinding(findings, "policy-minimal-devices", path) + self.assertFinding(findings, "policy-engine-rootless", path, level="warning") + # Linux capabilities pass as not-applicable only on a measured non-Linux family. + stated_family = changed(windows, lambda d: layer(d)["os"].update(family=fact("windows", provenance="declared"))) + self.assertFinding(check(stated_family), "policy-minimal-capabilities", "/workload/isolation/capabilities/effective") + self.assertFinding(check(changed(windows, lambda d: d["workload"]["isolation"]["capabilities"].update(effective=unknown()))), + "policy-minimal-capabilities", "/workload/isolation/capabilities/effective") + + def test_a_process_that_can_regain_root_fails(self): + result = fixture("windows-arm64-wsl2-podman.result") + identity = lambda d: d["workload"]["identity"] # noqa: E731 + for label, change, path in ( + ("real user ID 0", lambda d: identity(d).update(real_uid=fact(0, "synthetic:x")), "/workload/identity/real_uid"), + ("saved user ID 0", lambda d: identity(d).update(saved_uid=fact(0, "synthetic:x")), "/workload/identity/saved_uid"), + ("saved user ID unknown", lambda d: identity(d).update(saved_uid=unknown()), "/workload/identity/saved_uid"), + ("real user ID not applicable on Linux", lambda d: identity(d).update(real_uid={"value": None, "provenance": "not-applicable", "reason": "x"}), "/workload/identity/real_uid"), + ("permitted capability", lambda d: d["workload"]["isolation"]["capabilities"].update(permitted=fact(["cap_setuid"], "synthetic:x")), "/workload/isolation/capabilities/permitted"), + ("permitted capabilities unknown", lambda d: d["workload"]["isolation"]["capabilities"].update(permitted=unknown()), "/workload/isolation/capabilities/permitted"), + ): + with self.subTest(label): + findings = check(changed(result, change)) + self.assertEqual(len(findings.errors), 1, findings.items) + self.assertEqual(findings.errors[0][2], path) + + def test_an_exception_covers_only_what_it_names(self): + result, profile = pair("windows-arm64-wsl2-podman") + sudo_install = changed(result, lambda d: d["stages"]["install"]["privilege"].update(escalation=fact("sudo", provenance="declared"))) + self.assertCodes(check(sudo_install, profile), "policy-no-privilege-escalation") + approval = {"status": "approved", "reference": "https://github.com/P-R-N-D/Claus-AI (synthetic example)", "date": "2026-10-11"} + # The profile leaves these open, so that only the policy decides. + open_profile = changed(profile, lambda d: d["environment"]["layers"][2]["container"].update(privileged=unknown(), devices=unknown())) + + def allow(rule, **scope): + exception = {"id": f"allow-{rule}", "rule": rule, "reason": "x", "scope": scope, "verification": "x", "end_condition": "x", "approval": dict(approval)} + return changed(open_profile, lambda d: d["policy"].setdefault("exceptions", []).append(exception)) + + sudo_allowed = allow("no-privilege-escalation", stages=["install"], escalations=["sudo"]) + self.assertValid(check(sudo_allowed)) + findings = check(sudo_install, sudo_allowed) + self.assertValid(findings) + self.assertFinding(findings, "policy-exception-applied", "/policy/exceptions/0", level="warning") + for label, change in ( + ("another method", lambda d: d["stages"]["install"]["privilege"].update(escalation=fact("setuid", provenance="declared"))), + ("an unknown method", lambda d: d["stages"]["install"]["privilege"].update(escalation=unknown())), + ("another stage", lambda d: d["stages"]["execute"]["privilege"].update(escalation=fact("sudo", provenance="declared"))), + ): + with self.subTest(label): + self.assertIn("policy-no-privilege-escalation", check(changed(result, change), sudo_allowed).codes()) + unknown_method = check(changed(result, lambda d: d["stages"]["install"]["privilege"].update(escalation=unknown())), sudo_allowed) + self.assertIn("cannot cover a state that is not measured", unknown_method.errors[0][3]) + self.assertFinding(check(allow("no-privilege-escalation", stages=["install"])), "policy-exception", "/policy/exceptions/0/scope") + self.assertFinding(check(allow("no-privilege-escalation", escalations=["sudo"])), "policy-exception", "/policy/exceptions/0/scope") + # A layer exception covers a privileged container, not one whose state is unknown. + privileged_allowed = allow("no-privileged-container", layers=[2]) + privileged = changed(result, lambda d: d["environment"]["layers"][2]["container"].update(privileged=fact(True, "synthetic:x"))) + self.assertValid(check(privileged, privileged_allowed)) + unknown_state = changed(result, lambda d: d["environment"]["layers"][2]["container"].update(privileged=unknown())) + self.assertCodes(check(unknown_state, privileged_allowed), "policy-no-privileged-container") + # A device exception does not cover devices that are not known. + devices_allowed = allow("minimal-devices", devices=["fuse"]) + unknown_devices = changed(result, lambda d: d["environment"]["layers"][2]["container"].update(devices=unknown())) + self.assertCodes(check(unknown_devices, devices_allowed), "policy-minimal-devices") + # app-non-root needs no scope, but still cannot cover an unknown user. + root_allowed = allow("app-non-root") + root = changed(result, lambda d: d["workload"]["identity"].update(superuser=fact(True, "synthetic:x"), uid=fact(0, "synthetic:x"), real_uid=fact(0, "synthetic:x"), saved_uid=fact(0, "synthetic:x"))) + self.assertValid(check(root, root_allowed)) + self.assertCodes(check(changed(result, lambda d: d["workload"]["identity"].update(superuser=unknown())), root_allowed), "policy-app-non-root") + + def test_a_skipped_policy_is_reported(self): + findings = check(fixture("linux-x64-vm.result"), policy=False) + self.assertValid(findings) + self.assertIn("policy-skipped", findings.codes("warning")) + + +class ProfileTests(Case): + def test_support_claims_follow_the_dependency_strategy(self): + profile = fixture("windows-arm64-native.profile") + support = lambda d: d["support"] # noqa: E731 + for label, change in ( + ("synthetic checked claim", lambda d: support(d).update(status="checked")), + ("status that differs from the lane table", lambda d: support(d).update(status="statically-resolvable")), + ("lane that is not in the table", lambda d: support(d).update(lane="Windows on Arm, native")), + ("claim without a lane", lambda d: support(d).update(lane=None)), + ("not assessed with a lane that is not in the table", lambda d: support(d).update(status="not-assessed", lane="Windows on Arm")), + ): + with self.subTest(label): + self.assertIn("support-claim", check(changed(profile, change)).codes()) + unlisted = changed(profile, lambda d: support(d).update(status="not-assessed", lane=None)) + self.assertValid(check(unlisted)) + # A synthetic profile assesses nothing, so it may name its lane without a status from it. + self.assertValid(check(changed(profile, lambda d: support(d).update(status="not-assessed")))) + _, real = real_result() + self.assertFinding(check(changed(real, lambda d: d["support"].update(status="not-assessed"))), "support-claim", "/support/status") + # The macOS Apple Silicon row gives one status per macOS version: statically resolvable for 15 or newer and not + # supported for 14 or older. A claim needs the version that gives its status, and a real profile copies it. + mac = macos_profile() + version = lambda text: lambda d: d["environment"]["layers"][-1]["os"].update(version=fact(text, provenance="declared") if text else unknown()) # noqa: E731 + for status, os_version, valid in ( + ("statically-resolvable", "15.5", True), ("statically-resolvable", "15", True), ("statically-resolvable", "26.0.1", True), + ("statically-resolvable", "1000.1", True), + ("statically-resolvable", "14.7.1", False), ("statically-resolvable", None, False), ("statically-resolvable", "Sequoia", False), + ("not-supported", "14.7.1", True), ("not-supported", "13", True), ("not-supported", "15.0", False), ("not-supported", None, False), + ("checked", "15.5", False), ("not-assessed", None, True), ("not-assessed", "Sequoia", True), + ("not-assessed", "15.5", False), ("not-assessed", "14.7", False), ("statically-resolvable", "015.5", False), + ): + with self.subTest(status=status, os_version=os_version): + claimed = changed(mac, lambda d: (d["support"].update(status=status, lane="macOS Apple Silicon"), version(os_version)(d))) + findings = check(claimed) + self.assertEqual(not findings.errors, valid, findings.items) + if not valid and status != "checked": + path = "/support/status" if status == "not-assessed" else "/environment/layers/0/os/version" + self.assertEqual([(code, where) for _, code, where, _ in findings.errors], [("support-claim", path)]) + unread = check(changed(mac, lambda d: (d["support"].update(status="statically-resolvable", lane="macOS Apple Silicon"), version("Sequoia")(d)))) + self.assertEqual([message for _, _, _, message in unread.errors], ["lane 'macOS Apple Silicon' is statically-resolvable for macOS 15 or newer, so a statically-resolvable claim declares a macOS version, such as 14.7 or 15.5; this profile declares 'Sequoia'"]) + # The version is the innermost layer's: a macOS 15 guest on a macOS 14 host is macOS 15. + guest = changed(mac, lambda d: (d["support"].update(status="statically-resolvable", lane="macOS Apple Silicon"), version("15.5")(d))) + host = changed(guest["environment"]["layers"][0], lambda layer: layer.update(kind=fact("host", provenance="declared"), + technology={"value": None, "provenance": "not-applicable", "reason": "the outermost host runs on hardware"})) + host["os"].update(version=fact("14.7", provenance="declared")) + guest["environment"]["layers"][0].update(kind=fact("vm", provenance="declared"), technology=fact("other", provenance="declared")) + guest["environment"]["layers"].insert(0, host) + for status, valid in (("statically-resolvable", True), ("not-supported", False)): + with self.subTest(guest=status): + findings = check(changed(guest, lambda d: d["support"].update(status=status))) + self.assertEqual(not findings.errors, valid, findings.items) + # The Linux x86-64 glibc row is checked for glibc 2.39 or newer and CPython 3.12.13 or newer; a real profile + # below either bound, or that does not declare the version, names the lane with not-assessed. + libc = lambda text: lambda d: d["workload"]["runtimes"]["python"].update(libc=fact(text, provenance="declared") if text else unknown()) # noqa: E731 + cpython = lambda text: lambda d: d["workload"]["runtimes"]["python"].update(version=fact(text, provenance="declared")) # noqa: E731 + at_libc, at_version = ("support-claim", "/workload/runtimes/python/libc"), ("support-claim", "/workload/runtimes/python/version") + for label, change, checked_errors, not_assessed_valid in ( + ("glibc 2.39", libc("glibc 2.39"), [], False), + ("glibc 2.41", libc("glibc 2.41"), [], False), + ("glibc 2.39.1", libc("glibc 2.39.1"), [], False), + ("GLIBC 2.40", libc("GLIBC 2.40"), [], False), + ("glibc 2.34, RHEL 9", libc("glibc 2.34"), [at_libc], True), + ("glibc 1.39", libc("glibc 1.39"), [at_libc], True), + ("glibc without a version", libc("glibc"), [at_libc], True), + ("glibc from the ABI tag only", libc(None), [at_libc], True), + ("glibc 2.039, a leading zero", libc("glibc 2.039"), [at_libc], True), + ("glibc 02.39, a leading zero", libc("glibc 02.39"), [at_libc], True), + ("glibc and two spaces", libc("glibc 2.39"), [at_libc], True), + ("glibc 2.1000, four digits", libc("glibc 2.1000"), [], False), + ("CPython 3.12.13", cpython("3.12.13"), [], False), + ("CPython 3.12.12", cpython("3.12.12"), [at_version], True), + ("CPython 3.12.13rc1", cpython("3.12.13rc1"), [at_version], True), + ("CPython 3.12.14+", cpython("3.12.14+"), [at_version], True), + ("CPython 3.12", cpython("3.12"), [at_version], True), + ("CPython 3.12.1000, four digits", cpython("3.12.1000"), [], False), + ("CPython 3.12.10000", cpython("3.12.10000"), [at_version], True), + ): + with self.subTest(label): + claimed = changed(real, change) + self.assertEqual([(code, path) for _, code, path, _ in check(claimed).errors], checked_errors) + findings = check(changed(claimed, lambda d: d["support"].update(status="not-assessed"))) + self.assertEqual(not findings.errors, not_assessed_valid, findings.items) + message = [message for _, code, _, message in check(changed(real, libc("glibc 2.34"))).errors if code == "support-claim"][0] + self.assertEqual(message, "lane 'Linux x86-64 glibc' is checked for glibc 2.39 or newer; this profile declares 'glibc 2.34'") + # The lane table and the platform agree with this claim, so only the synthetic marking can reject it. + on_checked_lane = changed(fixture("linux-x64-vm.profile"), lambda d: (support(d).update(status="checked"), libc("glibc 2.39")(d))) + self.assertCodes(check(on_checked_lane), "support-claim") + self.assertEqual([path for _, code, path, _ in check(on_checked_lane).errors], ["/support/status"]) + + def test_support_claims_are_bound_to_the_lane_platform_and_interpreter(self): + _, real = real_result() + self.assertValid(check(real)) + python = lambda d: d["workload"]["runtimes"]["python"] # noqa: E731 + layers = lambda d: d["environment"]["layers"] # noqa: E731 + + def free_threaded_315(d): + python(d).update(version=fact("3.15.0", provenance="declared"), cache_tag=fact("cpython-315", provenance="declared"), + soabi=fact("cpython-315t-x86_64-linux-gnu", provenance="declared"), free_threaded_build=fact(True, provenance="declared"), + gil_enabled=fact(False, provenance="declared")) + + def aarch64(d): + layers(d)[1]["cpu"].update(arch=fact("aarch64", provenance="declared")) + python(d).update(arch=fact("aarch64", provenance="declared"), platform=fact("linux-aarch64", provenance="declared"), + soabi=fact("cpython-312-aarch64-linux-gnu", provenance="declared")) + d["workload"]["runtimes"]["node"].update(arch=fact("aarch64", provenance="declared")) + + for label, change, path in ( + ("Python 3.15 free-threaded", free_threaded_315, "/workload/runtimes/python"), + ("Python 3.13", lambda d: python(d).update(version=fact("3.13.9", provenance="declared"), cache_tag=fact("cpython-313", provenance="declared"), + soabi=fact("cpython-313-x86_64-linux-gnu", provenance="declared")), "/workload/runtimes/python"), + ("PyPy", lambda d: python(d).update(implementation=fact("pypy", provenance="declared"), cache_tag=unknown(), soabi=unknown()), "/workload/runtimes/python"), + ("interpreter not declared", lambda d: python(d).update(free_threaded_build=unknown()), "/workload/runtimes/python"), + ("another architecture", aarch64, "/workload/runtimes/python/arch"), + ("architecture not declared", lambda d: python(d).update(arch=unknown(), execution=unknown()), "/workload/runtimes/python/arch"), + ("musl", lambda d: python(d).update(libc=unknown(), soabi=fact("cpython-312-x86_64-linux-musl", provenance="declared")), "/workload/runtimes/python/soabi"), + ("C library not declared", lambda d: python(d).update(libc=unknown(), soabi=unknown()), "/workload/runtimes/python/soabi"), + ("another OS", lambda d: layers(d)[1]["os"].update(family=fact("freebsd", provenance="declared")), "/environment/layers/1/os/family"), + ): + with self.subTest(label): + claimed = changed(real, change) + self.assertFinding(check(claimed), "support-claim", path) + if label in ("Python 3.15 free-threaded", "Python 3.13", "PyPy"): + # The lane table does not assess this interpreter, so the profile records not-assessed. + self.assertValid(check(changed(claimed, lambda d: d["support"].update(status="not-assessed")))) + # A lane without version bounds covers every profile on its platform and interpreter, so a real one copies its status. + on_arm = changed(real, lambda d: (aarch64(d), d["support"].update(status="not-assessed", lane="Linux aarch64 glibc"))) + self.assertEqual([(code, path) for _, code, path, _ in check(on_arm).errors], [("support-claim", "/support/status")]) + self.assertValid(check(changed(on_arm, lambda d: d["support"].update(status="statically-resolvable")))) + # The C library comes from the libc fact before the ABI tag, and an error names that fact. glibc and musl are + # read in any letter case; a libc fact or ABI tag that names another C library contradicts the lane whatever the status. + at_libc, at_soabi = ["/workload/runtimes/python/libc"], ["/workload/runtimes/python/soabi"] + for status, lane, libc, abi_tag, errors in ( + ("not-supported", "Linux musl (Alpine)", "musl 1.2.5", None, []), + ("not-supported", "Linux musl (Alpine)", "Musl 1.2.5", None, []), + ("not-supported", "Linux musl (Alpine)", "musl1.2.5", None, []), + ("not-supported", "Linux musl (Alpine)", "muslx 1.2.5", None, at_libc), + ("not-supported", "Linux musl (Alpine)", "musl\u00e9 1.2.5", None, at_libc), + ("not-supported", "Linux musl (Alpine)", None, "cpython-312-arm-linux-musleabihf", []), + ("not-supported", "Linux musl (Alpine)", "glibc 2.39", None, at_libc), + ("not-supported", "Linux musl (Alpine)", "glibc 2.39", "cpython-312-x86_64-linux-gnu", at_libc), + ("not-assessed", "Linux x86-64 glibc", "uclibc-ng 1.0.45", None, at_libc), + ("not-assessed", "Linux x86-64 glibc", None, "cpython-312-x86_64-linux-android", at_soabi), + ): + with self.subTest(lane=lane, libc=libc, abi_tag=abi_tag): + claimed = changed(real, lambda d: (d["support"].update(status=status, lane=lane), python(d).update(libc=fact(libc, provenance="declared") if libc else unknown(), + soabi=fact(abi_tag, provenance="declared") if abi_tag else unknown()))) + self.assertEqual([path for _, code, path, _ in check(claimed).errors if code == "support-claim"], errors) + for libc, soabi, family in (("uclibc-ng 1.0.45", None, ("uclibc-ng 1.0.45", "/workload/runtimes/python/libc")), + ("GLIBC 2.39", None, ("glibc", "/workload/runtimes/python/libc")), + ("Ubuntu GLIBC 2.39", None, ("Ubuntu GLIBC 2.39", "/workload/runtimes/python/libc")), + ("glibcx 2.39", None, ("glibcx 2.39", "/workload/runtimes/python/libc")), + ("gl\u0131bc 2.39", None, ("gl\u0131bc 2.39", "/workload/runtimes/python/libc")), + ("glibc\u00e9 2.39", None, ("glibc\u00e9 2.39", "/workload/runtimes/python/libc")), + # A number that is not a letter, such as a superscript or a Roman numeral, may follow the name. + ("glibc\u00b2 2.39", None, ("glibc", "/workload/runtimes/python/libc")), + ("musl\u2160", None, ("musl", "/workload/runtimes/python/libc")), + (None, "cpython-312-arm-linux-musleabihf", ("musl", "/workload/runtimes/python/soabi")), + (None, "cpython-312-x86_64-linux-gnu_x", (None, None)), + (None, "cpython-312-aarch64-linux-android", ("android", "/workload/runtimes/python/soabi")), + (None, "cpython-312-arm-linux-gnueabihf", ("glibc", "/workload/runtimes/python/soabi"))): + with self.subTest(libc=libc, soabi=soabi): + facts = {"libc": fact(libc, provenance="declared") if libc else unknown(), "soabi": fact(soabi, provenance="declared") if soabi else unknown()} + self.assertEqual(validator.libc_family(facts), family) + # A long tag is searched in linear time; overlapping repeats take seconds here. + started = time.perf_counter() + self.assertIsNone(validator.libc_of_tag("cpython-312-x86_64-linux-" + "a" * 50000 + "-")) + self.assertLess(time.perf_counter() - started, 1) + # A WSL2 lane needs a WSL2 layer, and a native Windows lane none. + wsl = fixture("windows-arm64-wsl2-podman.profile") + self.assertValid(check(wsl)) + no_wsl = changed(wsl, lambda d: layers(d)[1].update(kind=fact("vm", provenance="declared"), technology=fact("hyper-v", provenance="declared"))) + self.assertFinding(check(no_wsl), "support-claim", "/environment/layers") + native = fixture("windows-arm64-native.profile") + self.assertFinding(check(changed(native, lambda d: python(d).update(arch=fact("x86_64", provenance="declared"), platform=fact("win-amd64", provenance="declared"), + soabi=fact("cp312-win_amd64", provenance="declared"), execution=unknown()))), + "support-claim", "/workload/runtimes/python/arch") + # A contradiction is an error even in a profile that claims no status. + self.assertFinding(check(changed(fixture("linux-x64-vm.profile"), aarch64)), "support-claim", "/workload/runtimes/python/arch") + + def test_the_validator_knows_every_lane_in_the_table(self): + self.assertEqual(set(validator.LANE_TARGETS), set(LANES)) + + def test_the_validator_keeps_the_version_bounds_of_the_table(self): + # Each bound the validator keeps is written in the sentence of its lane's backend lock cell that names the + # status, and each bound a cell writes is one the validator keeps for that lane. + cells = {} + for line in STRATEGY.read_text(encoding="utf-8").splitlines(): + parts = [part.strip() for part in line.strip().strip("|").split("|")] + if line.startswith("| ") and len(parts) >= 3 and parts[0] in LANES: + cells[parts[0]] = parts[2] + self.assertEqual(set(cells), set(LANES)) + words, kept = dict(validator.STATUS_WORDS), {} + for (lane, status), bounds in validator.STATUS_CONDITIONS.items(): + with self.subTest(lane=lane, status=status): + self.assertIn(status, LANES[lane]) + texts = [validator.bound_text(bound) for bound in bounds] + kept.setdefault(lane, set()).update(texts) + sentences = [sentence for sentence in re.split(r"\.\s+", cells[lane]) if re.search(rf"\b(?:{words[status]})\b", sentence, re.IGNORECASE)] + self.assertTrue(any(all(text in sentence for text in texts) for sentence in sentences), cells[lane]) + for lane, cell in cells.items(): + with self.subTest(lane=lane): + self.assertEqual(set(re.findall(r"\w+ [0-9][0-9.]* or (?:newer|older)\b", cell)), kept.get(lane, set())) + self.assertEqual(validator.bound_text(("CPython", (3, 12, 13), "or newer")), "CPython 3.12.13 or newer") + # Each bound names a version the validator reads, with as many parts as the bound compares. + _, real = real_result() + mac = changed(macos_profile(), lambda d: d["environment"]["layers"][-1]["os"].update(version=fact("15.5", provenance="declared"))) + for bound in {bound for bounds in validator.STATUS_CONDITIONS.values() for bound in bounds}: + with self.subTest(bound=validator.bound_text(bound)): + self.assertIsNotNone(validator.bound_met(mac if bound[0] == "macOS" else real, bound)) + + def test_a_checked_claim_needs_the_lane_table(self): + _, profile = real_result() + claimed = changed(profile, lambda d: d.update(support={"status": "checked", "lane": "Linux x86-64 glibc", "reference": "docs/DEPENDENCY-STRATEGY.md"})) + self.assertValid(check(claimed)) + self.assertIn("support-claim-unverified", check(claimed, lanes=None).codes("warning")) + other = changed(claimed, lambda d: d["support"].update(lane="Linux aarch64 glibc")) + self.assertCodes(check(other), "support-claim") + + def test_evidence_sources_name_a_real_workflow_job(self): + _, profile = real_result() + self.assertValid(check(profile)) + for label, source in ( + ("missing job", {"workflow": ".github/workflows/ci.yml", "job": "gpu"}), + ("missing workflow", {"workflow": ".github/workflows/devices.yml", "job": "compatibility"}), + ("outside the workflow directory", {"workflow": "docs/CONTEXT.md", "job": "compatibility"}), + ("path traversal", {"workflow": ".github/workflows/../../docs/CONTEXT.md", "job": "compatibility"}), + ("path traversal to a real workflow", {"workflow": ".github/workflows/../workflows/ci.yml", "job": "compatibility"}), + ("trigger key, not a job", {"workflow": ".github/workflows/ci.yml", "job": "pull_request"}), + ("step key, not a job", {"workflow": ".github/workflows/ci.yml", "job": "run"}), + ): + with self.subTest(label): + self.assertCodes(check(changed(profile, lambda d: d.update(evidence_sources=[source]))), "evidence-source") + traversal = changed(profile, lambda d: d.update(evidence_sources=[{"workflow": ".github/workflows/../workflows/ci.yml", "job": "compatibility"}])) + self.assertFinding(check(traversal), "evidence-source", "/evidence_sources/0/workflow") + unsourced = changed(profile, lambda d: d.update(evidence_sources=[])) + self.assertCodes(check(unsourced, real=True), "evidence-source") + self.assertFinding(check(unsourced), "evidence-source", "/evidence_sources") + # A synthetic profile is recorded by no job. + self.assertValid(check(changed(fixture("linux-x64-vm.profile"), lambda d: d.update(evidence_sources=[])))) + # A comment at the start of a line does not end the jobs block. + with tempfile.TemporaryDirectory() as directory: + workflow = pathlib.Path(directory) / ".github" / "workflows" / "ci.yml" + workflow.parent.mkdir(parents=True) + workflow.write_text("on: push\njobs:\n backend:\n runs-on: x\n# Compatibility\n compatibility:\n runs-on: x\n", encoding="utf-8") + with mock.patch.object(validator, "ROOT", pathlib.Path(directory)): + self.assertValid(check(profile)) + workflow.write_text("on: push\njobs:\n backend:\n runs-on: x\nenv:\n compatibility: x\n", encoding="utf-8") + self.assertCodes(check(profile), "evidence-source") + # A workflow that is not UTF-8, or a name longer than the system allows, is a finding, not a + # traceback that would print the path as written. + workflow.write_bytes(b"on: push\njobs:\n compatibility:\n runs-on: \xff\n") + self.assertFinding(check(profile), "evidence-source", "/evidence_sources/0/workflow") + self.assertIn("cannot read workflow file '.github/workflows/ci.yml' (UnicodeDecodeError)", check(profile).items[0][3]) + long = changed(profile, lambda d: d.update(evidence_sources=[{"workflow": ".github/workflows/##[error]" + "a" * 300 + ".yml", "job": "compatibility"}])) + self.assertFinding(check(long), "evidence-source", "/evidence_sources/0/workflow") + self.assertIn("cannot read workflow file", check(long).items[0][3]) + # A job is a key of the unindented jobs mapping at the indentation of its first key, plain or + # quoted, with a colon followed by a space, a tab or the end of the line. Lines end at "\n" only. + for label, text, defined in ( + ("four spaces", "on: push\njobs:\n backend:\n runs-on: x\n compatibility:\n runs-on: x\n", True), + ("a quoted jobs key", '"jobs":\n compatibility:\n runs-on: x\n', True), + ("a jobs key with a blank before the colon", "jobs :\n compatibility:\n runs-on: x\n", True), + ("a tab after the colon", "jobs:\t# all\n compatibility:\t# probe\n runs-on: x\n", True), + ("U+0085 inside a value", 'jobs:\n backend:\n name: "a\x85b"\n compatibility:\n runs-on: x\n', True), + ("U+2028 inside a value", 'jobs:\n backend:\n name: "x\u2028 compatibility: y"\n', False), + ("a comment right after the jobs colon", "jobs:#c:\n compatibility:\n runs-on: x\n", False), + ("a block scalar", "jobs: |\n compatibility: x\n", False), + ("a comment right after the job colon", "jobs:\n compatibility:#c\n", False), + ("double quotes", 'jobs:\n "compatibility":\n runs-on: x\n', True), + ("single quotes and a space", "jobs:\n 'compatibility' :\n runs-on: x\n", True), + ("comments after the keys", "jobs: # all\n compatibility: # schema and probe\n runs-on: x\n", True), + ("a deeper key", "jobs:\n backend:\n compatibility:\n x: 1\n", False), + ("a deeper key under four spaces", "jobs:\n backend:\n compatibility: 1\n", False), + ("a longer name", "jobs:\n compatibility-x:\n runs-on: x\n", False), + ("a name that ends with it", "jobs:\n pre-compatibility:\n runs-on: x\n", False), + ("no space after the colon", "jobs:\n compatibility:x\n", False), + ("mismatched quotes", "jobs:\n \"compatibility':\n runs-on: x\n", False), + ("an indented jobs key", "on:\n jobs:\n compatibility: x\n", False), + ("flow style", "jobs: {compatibility: {runs-on: x}}\n", False), + ): + with self.subTest(label): + workflow.write_text(text, encoding="utf-8") + if defined: + self.assertValid(check(profile)) + else: + self.assertFinding(check(profile), "evidence-source", "/evidence_sources/0/job") + # A job name is matched as written, so a dot in it matches only a dot. + dotted = changed(profile, lambda d: d.update(evidence_sources=[{"workflow": ".github/workflows/ci.yml", "job": "a.b"}])) + for text, defined in (("jobs:\n a.b:\n runs-on: x\n", True), ("jobs:\n axb:\n runs-on: x\n", False)): + with self.subTest(dotted=text): + workflow.write_text(text, encoding="utf-8") + self.assertEqual(check(dotted).codes(), [] if defined else ["evidence-source"]) + + def test_a_result_is_checked_against_what_its_profile_declares(self): + result, profile = pair("windows-arm64-wsl2-podman") + layers = lambda d: d["environment"]["layers"] # noqa: E731 + for label, change, code in ( + ("other profile id", lambda d: d.update(profile_id="synthetic-other"), "profile-link"), + ("other Python", lambda d: d["workload"]["runtimes"]["python"].update(version=fact("3.13.9", "synthetic:x"), cache_tag=fact("cpython-313", "synthetic:x"), soabi=fact("cpython-313-aarch64-linux-gnu", "synthetic:x")), "profile-mismatch"), + ("missing layer", lambda d: (layers(d).pop(1), layers(d)[0].update(kind=fact("host", "synthetic:x"))), "profile-mismatch"), + ("rootful engine", lambda d: layers(d)[2]["container"].update(engine_mode=fact("rootful", "synthetic:x")), "profile-mismatch"), + ("not established", lambda d: layers(d)[2]["os"].update(version=unknown()), "profile-unverified"), + ("backend missing", lambda d: d["compute"]["backends"][0].update(id="cpu0"), "profile-mismatch"), + ("backend of another kind", lambda d: d["compute"]["backends"][0].update(kind="other", api="wasm"), "profile-mismatch"), + ): + with self.subTest(label): + self.assertIn(code, check(changed(result, change), profile).codes()) + + def test_every_declared_fact_is_compared(self): + result, profile = pair("kubernetes-arm64-cuda") + self.assertValid(check(result, profile)) + layers = lambda d: d["environment"]["layers"] # noqa: E731 + cuda = lambda d: d["compute"]["backends"][1] # noqa: E731 + for label, change, path in ( + ("service manager scope", lambda d: layers(d)[0]["service_manager"].update(scope=fact("user", "synthetic:x")), "/environment/layers/0/service_manager/scope"), + ("orchestration version", lambda d: layers(d)[1]["container"]["orchestration"].update(version=fact("9.9", "synthetic:x")), "/environment/layers/1/container/orchestration/version"), + ("backend device", lambda d: cuda(d).update(device=fact("Other GPU", "synthetic:x")), "/compute/backends/1/device"), + ("backend version", lambda d: cuda(d).update(version=fact("12.0", "synthetic:x")), "/compute/backends/1/version"), + ("backend features", lambda d: cuda(d).update(features=fact(["fp16"], "synthetic:x")), "/compute/backends/1/features"), + ("accelerator inventory", lambda d: d["compute"].update(accelerator_inventory=fact(False, "synthetic:x")), "/compute/accelerator_inventory"), + ): + with self.subTest(label): + self.assertFinding(check(changed(result, change), profile), "profile-mismatch", path) + self.assertFinding(check(changed(result, lambda d: cuda(d).update(available=unknown())), profile), "profile-unverified", "/compute/backends/1/available") + # Extension objects are not facts, so vendor data the profile records is not compared. + self.assertValid(check(changed(result, lambda d: cuda(d).pop("extensions", None)), profile)) + + def test_layer_count_runtimes_and_backend_api_are_compared_on_their_own(self): + result, profile = pair("windows-arm64-wsl2-podman") + fewer = changed(result, lambda d: d["environment"]["layers"].pop()) + self.assertCodes(check(fewer, profile, policy=False), "profile-mismatch") + self.assertFinding(check(fewer, profile, policy=False), "profile-mismatch", "/environment/layers") + result, profile = pair("linux-x64-vm") + node = lambda d: d["workload"]["runtimes"]["node"] # noqa: E731 + for label, change, code, path in ( + ("node version", lambda d: node(d).update(version=fact("20.0.0", "synthetic:x")), "profile-mismatch", "/workload/runtimes/node/version"), + ("node modules ABI", lambda d: node(d).update(modules_abi=fact("115", "synthetic:x")), "profile-mismatch", "/workload/runtimes/node/modules_abi"), + ("node platform unknown", lambda d: node(d).update(platform=unknown()), "profile-unverified", "/workload/runtimes/node/platform"), + ): + with self.subTest(label): + self.assertFinding(check(changed(result, change), profile), code, path) + result, profile = pair("kubernetes-arm64-cuda") + other_api = changed(result, lambda d: d["compute"]["backends"][1].update(api="vulkan-compute")) + self.assertFinding(check(other_api, profile), "profile-mismatch", "/compute/backends/1") + + def test_only_measured_values_establish_what_a_profile_declares(self): + result, profile = pair("windows-arm64-wsl2-podman") + stated = changed(result, lambda d: d["environment"]["layers"][2]["container"].update(engine_mode=fact("rootless", provenance="declared"))) + self.assertFinding(check(stated, profile), "profile-unverified", "/environment/layers/2/container/engine_mode") + # This profile makes engine-rootless required. + self.assertFinding(check(stated, profile), "policy-engine-rootless", "/environment/layers/2/container/engine_mode") + + def test_a_fact_the_profile_calls_not_applicable_is_compared(self): + result, profile = pair("windows-arm64-native") + self.assertEqual(profile["workload"]["runtimes"]["python"]["libc"]["provenance"], "not-applicable") + glibc = changed(result, lambda d: d["workload"]["runtimes"]["python"].update(libc=fact("glibc 2.39", "synthetic:x"))) + self.assertFinding(check(glibc, profile), "profile-mismatch", "/workload/runtimes/python/libc") + + def test_the_runner_a_profile_declares_is_compared(self): + result, profile = real_result() + declared = lambda value: fact(value, provenance="declared") # noqa: E731 + profile = changed(profile, lambda d: d.update(runner={"runner_environment": declared("github-hosted"), "runner_os": declared("Linux"), "runner_arch": declared("X64")})) + self.assertValid(check(result, profile)) + for label, change, code in ( + ("self-hosted runner", lambda d: d["run"]["ci"].update(runner_environment=fact("self-hosted", "env:RUNNER_ENVIRONMENT")), "profile-mismatch"), + ("Arm runner", lambda d: d["run"]["ci"].update(runner_arch=fact("ARM64", "env:RUNNER_ARCH")), "profile-mismatch"), + ("runner not recorded", lambda d: d["run"]["ci"].update(runner_os=unknown()), "profile-unverified"), + ): + with self.subTest(label): + findings = check(changed(result, change), profile) + self.assertCodes(findings, code) + + def test_a_real_profile_accepts_only_github_actions_records(self): + result, profile = real_result() + for label, system in (("a local run", fact("local", "env:GITHUB_ACTIONS", provenance="inferred")), + ("a declared system", fact("github-actions", provenance="declared")), + ("an unknown system", unknown())): + with self.subTest(label): + findings = check(changed(result, lambda d: d["run"]["ci"].update(system=system)), profile) + self.assertFinding(findings, "profile-link", "/run/ci/system") + + def test_a_real_result_is_linked_to_a_real_profile_and_its_job(self): + result, profile = real_result() + self.assertValid(check(result, profile)) + self.assertCodes(check(result, fixture("linux-x64-vm.profile")), "profile-link") + # Same id and job, so only the synthetic flag differs. + synthetic = changed(fixture("linux-x64-vm.profile"), lambda d: d.update(id=result["profile_id"], evidence_sources=profile["evidence_sources"])) + self.assertCodes(check(result, synthetic), "profile-link") + self.assertFinding(check(result, synthetic), "profile-link", "/synthetic") + other_job = changed(result, lambda d: d["run"]["ci"].update(job=fact("backend", "env:GITHUB_JOB"))) + self.assertCodes(check(other_job, profile), "profile-link") + # The job must also be in the workflow the profile names. + other_workflow = changed(result, lambda d: d["run"]["ci"].update(workflow_ref=fact("P-R-N-D/Claus-AI/.github/workflows/other.yml@refs/pull/1/merge", "env:GITHUB_WORKFLOW_REF"))) + self.assertCodes(check(other_workflow, profile), "profile-link") + # A known workflow ref in another repository is a mismatch, not a missing ref, even when the job matches. + for ref in ("Other/Repo/.github/workflows/ci.yml@refs/heads/main", "P-R-N-D/Claus-AI-fork/.github/workflows/ci.yml@refs/heads/main", + "P-R-N-D/Claus-AIX.github/workflows/ci.yml@refs/heads/main"): + with self.subTest(workflow_ref=ref): + elsewhere = changed(result, lambda d: d["run"]["ci"].update(workflow_ref=fact(ref, "env:GITHUB_WORKFLOW_REF"))) + self.assertCodes(check(elsewhere, profile), "profile-link") + self.assertFinding(check(elsewhere, profile), "profile-link", "/run/ci/workflow_ref") + self.assertFinding(check(other_workflow, profile), "profile-link", "/run/ci/workflow_ref") + # A ref that names no workflow file or no ref links to no workflow. + for ref in ("P-R-N-D/Claus-AI/.github/workflows/ci.yml", "P-R-N-D/Claus-AI/@refs/pull/1/merge", "P-R-N-D/Claus-AI/.github/workflows/ci.yml@"): + with self.subTest(workflow_ref=ref): + partial = changed(result, lambda d: d["run"]["ci"].update(workflow_ref=fact(ref, "env:GITHUB_WORKFLOW_REF"))) + self.assertEqual(check(partial, profile).errors, [("error", "profile-link", "/run/ci/workflow_ref", f"the workflow ref {ref!r} does not name a workflow file and a ref")]) + # The ref is what follows the first "@", and git allows "@" in a branch name. + at_sign = changed(result, lambda d: d["run"]["ci"].update(workflow_ref=fact("P-R-N-D/Claus-AI/.github/workflows/ci.yml@refs/heads/a@b", "env:GITHUB_WORKFLOW_REF"))) + self.assertValid(check(at_sign, profile)) + # A run is linked by its workflow and its job, so a record without its workflow ref or its repository links to + # no profile, even with the profile's job, and fails as CI evidence too. The finding names what is missing. + for label, change, path, missing in (("no workflow ref", lambda d: d["run"]["ci"].update(workflow_ref=unknown()), "/run/ci/workflow_ref", "workflow ref"), + ("no repository", lambda d: d["run"]["ci"].update(repository=unknown()), "/run/ci/repository", "repository"), + ("neither", lambda d: d["run"]["ci"].update(workflow_ref=unknown(), repository=unknown()), "/run/ci/workflow_ref", "repository and workflow ref")): + with self.subTest(label): + findings = check(changed(result, change), profile) + self.assertCodes(findings, "ci-evidence", "profile-link") + message = f"a GitHub Actions record is linked to profile {profile['id']} by its repository, workflow ref and job; this one does not name its {missing}" + self.assertEqual([item for item in findings.errors if item[1] == "profile-link"], [("error", "profile-link", path, message)]) + self.assertFinding(check(changed(other_job, lambda d: d["run"]["ci"].update(workflow_ref=unknown())), profile), "profile-link", "/run/ci/workflow_ref") + + def test_a_github_actions_record_names_its_run_and_commits(self): + result, profile = real_result() + ci = lambda d: d["run"]["ci"] # noqa: E731 + revision = lambda d: d["run"]["revision"] # noqa: E731 + for label, change in ( + ("no run id", lambda d: ci(d).update(run_id=unknown())), + ("run URL of another repository", lambda d: ci(d).update(run_url=fact("https://github.com/other/repo/actions/runs/100", "env:x", provenance="inferred"))), + ("no GITHUB_SHA", lambda d: revision(d).update(event_sha=unknown())), + ("no checked-out commit", lambda d: revision(d).update(tested_sha=unknown())), + ("pull request without its head", lambda d: revision(d).update(pr_head_sha=unknown())), + ("pull request without its base", lambda d: revision(d).update(pr_base_sha=unknown())), + ("no run attempt", lambda d: ci(d).update(run_attempt=unknown())), + ("run attempt 0", lambda d: ci(d).update(run_attempt=fact(0, "env:GITHUB_RUN_ATTEMPT"))), + ("run URL of another run", lambda d: ci(d).update(run_url=fact("https://github.com/P-R-N-D/Claus-AI/actions/runs/101", "env:x", provenance="inferred"))), + ("run URL of another attempt", lambda d: ci(d).update(run_url=fact("https://github.com/P-R-N-D/Claus-AI/actions/runs/100/attempts/2", "env:x", provenance="inferred"))), + ("pull_request_target without the pull request", lambda d: (ci(d).update(event=fact("pull_request_target", "env:GITHUB_EVENT_NAME")), + revision(d).update(pr_head_sha=unknown(), pr_base_sha=unknown()))), + ): + with self.subTest(label): + self.assertCodes(check(changed(result, change), profile), "ci-evidence") + # A declared run or commit is a claim the record cannot back, so each must be observed or inferred. + for group, names in ((ci, ("event", "repository", "run_id", "run_attempt", "run_url", "workflow_ref", "job")), (revision, ("tested_sha", "event_sha", "pr_head_sha", "pr_base_sha"))): + for name in names: + with self.subTest(declared=name): + path = f"/run/{'ci' if group is ci else 'revision'}/{name}" + declared = changed(result, lambda d: group(d).update({name: fact(group(d)[name]["value"], provenance="declared")})) + self.assertFinding(check(declared, profile), "ci-evidence", path) + self.assertFinding(check(changed(result, lambda d: ci(d).update(job=unknown())), profile), "ci-evidence", "/run/ci/job") + self.assertFinding(check(changed(result, lambda d: ci(d).update(workflow_ref=unknown())), profile), "ci-evidence", "/run/ci/workflow_ref") + # A URL with no attempt names the run, whatever its attempt; one with an attempt names that attempt. + attempt = lambda n, url: lambda d: ci(d).update(run_attempt=fact(n, "env:GITHUB_RUN_ATTEMPT"), run_url=fact(url, "env:x", provenance="inferred")) # noqa: E731 + run = "https://github.com/P-R-N-D/Claus-AI/actions/runs/100" + self.assertValid(check(changed(result, attempt(2, run)), profile)) + self.assertValid(check(changed(result, attempt(2, run + "/attempts/2")), profile)) + self.assertFinding(check(changed(result, attempt(2, run + "/attempts/1")), profile), "ci-evidence", "/run/ci/run_url") + # An attempt longer than any run_attempt names no run, also under a schema that lets such a URL through. + loose = copy.deepcopy(SCHEMA) + loose["$defs"]["fact_string"]["properties"]["value"]["maxLength"] = 10000 + long_attempt = changed(result, attempt(2, run + "/attempts/" + "1" * 5000)) + self.assertFinding(validator.validate_document(long_attempt, loose, profile, LANES), "ci-evidence", "/run/ci/run_url") + # The bound is the schema's: the largest run_attempt is read, an 11-digit attempt names no run. + self.assertValid(check(changed(result, attempt(4294967295, run + "/attempts/4294967295")), profile)) + eleven = check(changed(result, attempt(2, run + "/attempts/" + "1" * 11)), profile) + self.assertIn(("error", "ci-evidence", "/run/ci/run_url", "the run URL does not point at this repository's run"), eleven.items) + self.assertFinding(check(changed(result, lambda d: ci(d).update(run_attempt=fact(0, "env:GITHUB_RUN_ATTEMPT"))), profile), "ci-evidence", "/run/ci/run_attempt") + push = changed(result, lambda d: (ci(d).update(event=fact("push", "env:GITHUB_EVENT_NAME")), + revision(d).update(pr_head_sha={"value": None, "provenance": "not-applicable", "reason": "push"}, + pr_base_sha={"value": None, "provenance": "not-applicable", "reason": "push"}))) + self.assertValid(check(push, profile)) + + def test_strategy_lane_table_parsing(self): + self.assertEqual(LANES["Linux x86-64 glibc"], {"checked"}) + self.assertEqual(LANES["Windows on Arm, native (`win_arm64`)"], {"not-supported"}) + self.assertEqual(LANES["Windows on Arm with WSL2 (Ubuntu arm64)"], {"statically-resolvable"}) + self.assertEqual(LANES["macOS Apple Silicon"], {"statically-resolvable", "not-supported"}) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "strategy.md" + path.write_text("# T\n\n## Platform and accelerator lanes\n\n| Lane | Wheels | Backend lock | Notes |\n|---|---|---|---|\n" + "| A | x | Checked: CI | n |\n| B | x | Same as A | n |\n| C | x | Partly | n |\n" + "| D | x | v2: statically resolvable. v1: not supported (no wheel) | n |\n", encoding="utf-8") + self.assertEqual(validator.read_strategy_lanes(path), {"A": {"checked"}, "B": {"checked"}, "C": set(), "D": {"statically-resolvable", "not-supported"}}) + path.write_text("# T\n\n## Other\n", encoding="utf-8") + with self.assertRaises(ValueError): + validator.read_strategy_lanes(path) + # A lane listed twice would leave only its last status; the table is then ambiguous. + path.write_text("# T\n\n## Platform and accelerator lanes\n\n| Lane | Wheels | Backend lock | Notes |\n|---|---|---|---|\n" + "| A | x | Checked: CI | n |\n| A | x | Not supported | n |\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "lists 'A' twice"): + validator.read_strategy_lanes(path) + + +class CommandTests(Case): + def run_main(self, *argv): + with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()) as err: + try: + code = validator.main([str(arg) for arg in argv]) + except SystemExit as exc: + code = exc.code + return code, out.getvalue() + err.getvalue() + + def test_exit_status(self): + result, profile = FIXTURES / "linux-x64-vm.result.json", FIXTURES / "linux-x64-vm.profile.json" + self.assertEqual(self.run_main("--dependency-strategy", STRATEGY, "--profile", profile, result, profile)[0], 0) + self.assertEqual(self.run_main("--require-real", result)[0], 1) + with tempfile.TemporaryDirectory() as directory: + directory = pathlib.Path(directory) + invalid = directory / "invalid.json" + invalid.write_text(json.dumps(changed(fixture("linux-x64-vm.result"), lambda d: d.update(overall="passed"))), encoding="utf-8") + code, output = self.run_main(invalid) + self.assertEqual(code, 1) + self.assertIn("ERROR overall-mismatch /overall", output) + self.assertEqual(self.run_main(directory / "missing.json")[0], 2) + self.assertEqual(self.run_main("--profile", invalid, result)[0], 2) + self.assertEqual(self.run_main("--profile", result, result)[0], 2) + bad_schema = directory / "schema.json" + bad_schema.write_text(json.dumps({"oneOf": []}), encoding="utf-8") + self.assertEqual(self.run_main("--schema", bad_schema, result)[0], 2) + no_lanes = directory / "strategy.md" + no_lanes.write_text("# Nothing\n", encoding="utf-8") + self.assertEqual(self.run_main("--dependency-strategy", no_lanes, result)[0], 2) + + def test_an_unwritable_summary_is_a_usage_error(self): + with tempfile.TemporaryDirectory() as directory: + code, output = self.run_main("--summary", pathlib.Path(directory) / "missing" / "summary.md", FIXTURES / "linux-x64-vm.profile.json") + self.assertEqual(code, 2) + self.assertIn("cannot write the summary", output) + + def test_a_summary_that_is_a_file_this_run_reads_is_a_usage_error(self): + # The summary is appended to, so naming an input, by any spelling or link, would change that input. + with tempfile.TemporaryDirectory() as directory: + directory = pathlib.Path(directory) + sources = {"result": FIXTURES / "linux-x64-vm.result.json", "profile": FIXTURES / "linux-x64-vm.profile.json", "schema": validator.DEFAULT_SCHEMA, "strategy": STRATEGY} + copies = {name: directory / f"{name}{path.suffix}" for name, path in sources.items()} + for name, path in sources.items(): + copies[name].write_bytes(path.read_bytes()) + argv = ("--schema", copies["schema"], "--profile", copies["profile"], "--dependency-strategy", copies["strategy"], copies["result"]) + self.assertEqual(self.run_main(*argv, "--summary", directory / "summary.md")[0], 0) + (directory / "linked").hardlink_to(copies["result"]) + for name, summary in (*copies.items(), ("hard link", directory / "linked"), ("another spelling", directory / ".." / directory.name / "result.json")): + with self.subTest(summary=name): + before = {path: path.read_bytes() for path in copies.values()} + code, output = self.run_main(*argv, "--summary", summary) + self.assertEqual(code, 2) + self.assertIn("is the schema, the profile, the lane table or a document of this run", output) + self.assertEqual({path: path.read_bytes() for path in copies.values()}, before) + + def test_a_summary_that_is_an_evidence_workflow_is_a_usage_error(self): + # A real profile's evidence sources are read from its workflow files, so a summary appended there would change one. + result, profile = real_result() + # Its files count whenever check_evidence_sources reads them, also under a schema that lets `synthetic` be + # something other than a boolean. + for synthetic, count in ((False, 1), (0, 1), (None, 1), (True, 0), (1, 0)): + self.assertEqual(len(validator.evidence_workflows(dict(profile, synthetic=synthetic))), count) + with tempfile.TemporaryDirectory() as directory: + directory = pathlib.Path(directory) + workflow = directory / ".github" / "workflows" / "ci.yml" + workflow.parent.mkdir(parents=True) + workflow.write_bytes((validator.ROOT / ".github" / "workflows" / "ci.yml").read_bytes()) + (directory / "linked.yml").hardlink_to(workflow) + paths = {name: directory / f"{name}.json" for name in ("result", "profile")} + paths["result"].write_text(json.dumps(result), encoding="utf-8") + paths["profile"].write_text(json.dumps(profile), encoding="utf-8") + before = workflow.read_bytes() + with mock.patch.object(validator, "ROOT", directory): + self.assertEqual(self.run_main("--profile", paths["profile"], paths["result"], "--summary", directory / "summary.md")[0], 0) + for label, argv in (("profile", ("--profile", paths["profile"], paths["result"])), ("document", (paths["profile"],))): + for summary in (workflow, directory / "linked.yml"): + with self.subTest(label, summary=summary.name): + workflow.write_bytes(before) + code, output = self.run_main(*argv, "--summary", summary) + self.assertEqual(code, 2) + self.assertIn("is a workflow file a profile of this run names as its evidence source", output) + self.assertEqual(workflow.read_bytes(), before) + # A workflow file that is missing is refused too, however its path is written, and is not created. + missing = directory / ".github" / "workflows" / "missing.yml" + paths["missing"] = directory / "missing.json" + paths["missing"].write_text(json.dumps(changed(profile, lambda d: d.update(evidence_sources=[{"workflow": ".github/workflows/missing.yml", "job": "compatibility"}]))), encoding="utf-8") + spellings = [missing, directory / ".github" / "workflows" / ".." / "workflows" / "missing.yml"] + try: + (directory / "dangling.md").symlink_to(missing) + spellings.append(directory / "dangling.md") + except OSError: + pass + for summary in spellings: + with self.subTest("missing workflow", summary=str(summary.relative_to(directory))): + code, output = self.run_main(paths["missing"], "--summary", summary) + self.assertEqual(code, 2) + self.assertIn("is a workflow file a profile of this run names as its evidence source", output) + self.assertFalse(missing.exists()) + # Any other summary path is written: the profile fails for its missing workflow, and the summary says so. + elsewhere = directory / "workflows" / "missing.yml" + elsewhere.parent.mkdir() + code, output = self.run_main(paths["missing"], "--summary", elsewhere) + self.assertEqual(code, 1, output) + self.assertIn("evidence-source", elsewhere.read_text(encoding="utf-8")) + self.assertFalse(missing.exists()) + # A summary path that loops is a usage error, not a traceback, also where resolving it raises. + try: + (directory / "loop.md").symlink_to(directory / "loop.md") + except OSError: + pass + else: + code, output = self.run_main(paths["profile"], "--summary", directory / "loop.md") + self.assertEqual(code, 2) + self.assertIn("cannot write the summary", output) + + def test_argparse_errors_cannot_forge_output_lines(self): + # argparse repeats an argument it rejects, and the program name, in its usage line too; both are written as every other line is. + token = "ghp_" + "A" * 36 + with mock.patch.object(sys, "argv", ["::program##[error]forged"]): + code, output = self.run_main("--bad=##[error]forged", "--x=a\n::error::forged", f"--worse={token}", FIXTURES / "linux-x64-vm.result.json") + self.assertEqual(code, 2) + self.assertIn("unrecognized arguments", output) + self.assertIn("looks like a GitHub token", output) + self.assertNotIn("##[", output) + self.assertNotIn(token, output) + self.assertFalse([line for line in output.splitlines() if line.lstrip().startswith("::")], output) + self.assertIn("usage:", output) + + def test_document_text_cannot_forge_output_lines(self): + with tempfile.TemporaryDirectory() as directory: + directory = pathlib.Path(directory) + for label, tag in (("newline", "cpython-31\n::error title=x::forged"), ("lone surrogate", "cpython-31\ud800")): + with self.subTest(label): + path = directory / "result.json" + path.write_text(json.dumps(changed(fixture("linux-x64-vm.result"), lambda d: d["workload"]["runtimes"]["python"].update(cache_tag=fact(tag, "synthetic:x")))), encoding="utf-8") + summary = directory / "summary.md" + code, output = self.run_main("--summary", summary, path) + self.assertEqual(code, 1) + self.assertIn("control-character", output) + self.assertFalse([line for line in output.splitlines() if line.lstrip().startswith("::")], output) + self.assertNotRegex(output + summary.read_text(encoding="utf-8"), "[\ud800-\udfff]") + + def test_document_text_cannot_start_a_legacy_workflow_command(self): + # The runner reads "##[name]" anywhere in a line, so the value is echoed with "##[" broken up. + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "result.json" + version = "24.04 ##[stop-commands]token" + path.write_text(json.dumps(changed(fixture("linux-x64-vm.result"), lambda d: d["environment"]["layers"][1]["os"].update(version=fact(version, "synthetic:x")))), encoding="utf-8") + code, output = self.run_main("--profile", FIXTURES / "linux-x64-vm.profile.json", path) + self.assertEqual(code, 1) + self.assertIn("profile-mismatch", output) + self.assertIn("24.04 ## [stop-commands]token", output) + self.assertNotIn("##[", output) + + def test_secret_shaped_values_are_not_shown(self): + # A value the schema rejects is echoed in its finding, and a file name is printed; neither may show a token. + # The echo may escape the character before the value (repr's \n, a JSON Pointer's ~1), which a leading \b + # in a shape would miss. The summary escapes "_", so it is checked for the token's body. + token = "gh" + "p_" + "A" * 36 + documents = { + "kind": changed(fixture("linux-x64-vm.result"), lambda d: d.update(kind=token)), + "escaped": changed(fixture("linux-x64-vm.result"), lambda d: d.update(kind="\n" + token)), + "pointer": changed(fixture("linux-x64-vm.result"), lambda d: d.update({"/" + token: 1})), + } + with tempfile.TemporaryDirectory() as directory: + for case, document, name in [(case, document, "record.json") for case, document in documents.items()] + [("name", documents["kind"], token + ".json")]: + with self.subTest(case=case): + path = pathlib.Path(directory) / name + path.write_text(json.dumps(document), encoding="utf-8") + summary = pathlib.Path(directory) / "summary.md" + summary.write_text("", encoding="utf-8") + code, output = self.run_main("--summary", summary, path) + summary = summary.read_text(encoding="utf-8") + self.assertEqual(code, 1) + self.assertIn("not shown: looks like a GitHub token", output) + self.assertIn("not shown: looks like a GitHub token", summary) + self.assertNotIn("A" * 36, output + summary) + self.assertEqual(validator.plain("a " + token + " b ::c"), "a [not shown: looks like a GitHub token]") + self.assertEqual(validator.plain(repr("\x07" + token)), "'\\x07[not shown: looks like a GitHub token]") + + def test_file_names_cannot_forge_output_lines(self): + with tempfile.TemporaryDirectory() as directory: + missing = pathlib.Path(directory) / "x\n::error title=forged::y.json" + for argv in ((missing,), ("--profile", missing, FIXTURES / "linux-x64-vm.result.json"), ("--schema", missing, FIXTURES / "linux-x64-vm.result.json")): + with self.subTest(argv=argv[0]): + code, output = self.run_main(*argv) + self.assertEqual(code, 2) + self.assertFalse([line for line in output.splitlines() if line.lstrip().startswith("::")], output) + # A relative file name is printed as given, so one that starts with "::" would start the line. + name = "::warning title=forged::x.json" + (pathlib.Path(directory) / name).write_bytes((FIXTURES / "linux-x64-vm.result.json").read_bytes()) + with contextlib.chdir(directory): + code, output = self.run_main(name) + self.assertEqual(code, 0) + self.assertIn("valid result", output) + self.assertFalse([line for line in output.splitlines() if line.lstrip().startswith("::")], output) + + def test_an_unusable_schema_is_a_usage_error(self): + with tempfile.TemporaryDirectory() as directory: + schema = pathlib.Path(directory) / "schema.json" + for bad in ([], {"type": [["object"]]}, {"$defs": {"a": {"then": {"$ref": "#/$defs/a"}}}}): + with self.subTest(schema=bad): + schema.write_text(json.dumps(bad), encoding="utf-8") + code, output = self.run_main("--schema", schema, FIXTURES / "linux-x64-vm.result.json") + self.assertEqual(code, 2) + self.assertIn("cannot use schema", output) + + def test_a_schema_that_accepts_what_the_rules_cannot_read_is_a_usage_error(self): + # The rules across fields read the shape the default schema requires; a looser schema is no verdict. + with tempfile.TemporaryDirectory() as directory: + directory = pathlib.Path(directory) + schema, document = directory / "schema.json", directory / "record.json" + schema.write_text(json.dumps({"$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object"}), encoding="utf-8") + for content in ({}, {"kind": "result", "run": {}}, {"kind": "profile"}): + with self.subTest(content=content): + document.write_text(json.dumps(content), encoding="utf-8") + for argv in (("--schema", schema, document), ("--schema", schema, "--profile", document, FIXTURES / "linux-x64-vm.result.json")): + code, output = self.run_main(*argv) + self.assertEqual(code, 2) + self.assertIn("accepts a document the rules across fields cannot read", output) + self.assertNotIn("Traceback", output) + + def test_a_rule_that_fails_under_the_default_schema_is_not_hidden(self): + # With the default schema, however its path is written, an exception in a rule is a bug, not a usage error. + result = FIXTURES / "linux-x64-vm.result.json" + with mock.patch.object(validator, "check_strings", side_effect=KeyError("injected")), contextlib.chdir(validator.ROOT): + for argv in ((result,), ("--schema", validator.DEFAULT_SCHEMA, result), ("--schema", validator.DEFAULT_SCHEMA.relative_to(validator.ROOT), result)): + with self.subTest(argv=argv[:2]), self.assertRaises(KeyError): + self.run_main(*argv) + + def test_summary_lists_at_most_200_findings(self): + with tempfile.TemporaryDirectory() as directory: + directory = pathlib.Path(directory) + path = directory / "many.json" + path.write_text(json.dumps(changed(fixture("linux-x64-vm.result"), lambda d: d["extensions"].update({"x-many": {f"k{i}": "a\nb" for i in range(250)}}))), encoding="utf-8") + summary = directory / "summary.md" + code, _ = self.run_main("--summary", summary, path) + self.assertEqual(code, 1) + lines = summary.read_text(encoding="utf-8").splitlines() + self.assertEqual(sum(line.startswith("- error") for line in lines), 200) + self.assertIn("- 51 more findings are in the job log", lines) + + def test_a_skipped_policy_is_visible(self): + code, output = self.run_main("--no-policy", FIXTURES / "linux-x64-vm.result.json") + self.assertEqual(code, 0) + self.assertIn("WARNING policy-skipped /", output) + + def test_a_profile_whose_approval_is_incomplete_is_not_usable(self): + _, profile = pair("kubernetes-arm64-cuda") + result = FIXTURES / "kubernetes-arm64-cuda.result.json" + with tempfile.TemporaryDirectory() as directory: + for field in ("date", "reference"): + with self.subTest(missing=field): + incomplete = changed(profile, lambda d: d["policy"]["exceptions"][0]["approval"].pop(field)) + self.assertFinding(check(incomplete), "policy-exception", "/policy/exceptions/0/approval") + path = pathlib.Path(directory) / "profile.json" + path.write_text(json.dumps(incomplete), encoding="utf-8") + code, output = self.run_main("--profile", path, result) + self.assertEqual(code, 2) + self.assertIn("is not a valid profile", output) + + def test_summary_cells_escape_markdown(self): + self.assertEqual(validator.markdown_cell("a|b`c\nd"), "a\\|b\\`c d") + self.assertEqual(validator.markdown_cell("‮"), "\\‮") + self.assertEqual(validator.markdown_cell("[x](https://example.invalid) \\"), "\\[x\\]\\(https://example.invalid\\) \\\\\\") + + def test_summary_states_what_a_valid_record_means(self): + with tempfile.TemporaryDirectory() as directory: + directory = pathlib.Path(directory) + odd = directory / "a|b.json" + odd.write_text(json.dumps(changed(fixture("linux-x64-vm.result"), lambda d: d.update(overall="passed"))), encoding="utf-8") + summary = directory / "summary.md" + code, _ = self.run_main("--summary", summary, FIXTURES / "linux-x64-vm.result.json", odd) + self.assertEqual(code, 1) + text = summary.read_text(encoding="utf-8") + self.assertIn("| linux-x64-vm.result.json | result | valid | 0 | 1 |", text) + self.assertIn("| a\\|b.json | result | invalid | 1 | 1 |", text) + self.assertIn("It is not a support claim", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/tests/test_environment_probe.py b/scripts/ci/tests/test_environment_probe.py new file mode 100644 index 0000000..e2658ca --- /dev/null +++ b/scripts/ci/tests/test_environment_probe.py @@ -0,0 +1,1732 @@ +"""Tests for scripts/ci/collect_environment.py. + +Most tests point the probe at a made-up root directory, so they describe +machines the test does not run on: a GitHub-hosted runner VM, a container in a +VM, a rootless Podman container, WSL2, an emulated process, a Kubernetes pod. +Other tests run the collector on this machine in a subprocess, with secret-like +environment variables set, and check that none of them reach the record. No +Django and no third-party package is needed. From the repository root: + python -I scripts/ci/tests/test_environment_probe.py --verbose +""" + +import ast +import contextlib +import importlib.util +import io +import itertools +import json +import os +import pathlib +import re +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +import types +import unittest +from unittest import mock + +ROOT = pathlib.Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "ci" / "collect_environment.py" +VALIDATOR = ROOT / "scripts" / "ci" / "validate_compatibility.py" + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +collector = load("collect_environment", SCRIPT) +validator = load("validate_compatibility", VALIDATOR) +SCHEMA = validator.load_schema() + +FULL = "000001ffffffffff" +CPUINFO_X86 = "processor\t: 0\nvendor_id\t: AuthenticAMD\nflags\t\t: fpu vme sse sse2 {hypervisor}\n\nprocessor\t: 1\n" +CPUINFO_ARM = "processor\t: 0\nBogoMIPS\t: 50.00\nFeatures\t: fp asimd\nCPU implementer\t: 0x41\n" + + +def status(uid=1001, gid=118, cap_eff="0000000000000000", cap_bnd=FULL, no_new_privs=0, seccomp=0, real_uid=None, saved_uid=None, cap_prm=None): + real_uid = uid if real_uid is None else real_uid + saved_uid = uid if saved_uid is None else saved_uid + cap_prm = cap_eff if cap_prm is None else cap_prm + return ( + f"Name:\tpython\nUid:\t{real_uid}\t{uid}\t{saved_uid}\t{uid}\nGid:\t{gid}\t{gid}\t{gid}\t{gid}\n" + f"CapInh:\t0000000000000000\nCapPrm:\t{cap_prm}\nCapEff:\t{cap_eff}\nCapBnd:\t{cap_bnd}\n" + f"CapAmb:\t0000000000000000\nNoNewPrivs:\t{no_new_privs}\nSeccomp:\t{seccomp}\n" + ) + + +def mountinfo(*mounts): + lines = ["22 1 8:1 / / rw,relatime - ext4 /dev/root rw"] + lines += [f"{30 + index} 22 0:{30 + index} / {point} rw,nosuid - {fstype} {fstype} rw" for index, (point, fstype) in enumerate(mounts)] + return "\n".join(lines) + "\n" + + +RUNNER_FILES = { + "proc/cpuinfo": CPUINFO_X86.format(hypervisor="hypervisor"), + "proc/self/status": status(), + "proc/self/uid_map": " 0 0 4294967295\n", + "proc/self/cgroup": "0::/system.slice/runner.service\n", + "proc/self/mountinfo": mountinfo(("/sys/fs/cgroup", "cgroup2")), + "proc/self/attr/apparmor/current": "unconfined\n", + "proc/1/comm": "systemd\n", + "proc/sys/kernel/osrelease": "6.17.0-1022-azure\n", + "proc/sys/kernel/arch": "x86_64\n", + "proc/filesystems": "nodev\tsysfs\nnodev\tcgroup2\n", + "etc/os-release": 'NAME="Ubuntu"\nVERSION_ID="24.04"\nID=ubuntu\nPRETTY_NAME="Ubuntu 24.04.5 LTS"\n', + "sys/class/dmi/id/sys_vendor": "Microsoft Corporation\n", + "sys/class/dmi/id/product_name": "Virtual Machine\n", + "sys/module/apparmor/parameters/enabled": "Y\n", + "sys/kernel/security/lsm": "lockdown,capability,landlock,yama,apparmor,ima,evm", + "sys/fs/cgroup/cgroup.controllers": "cpu memory pids\n", + "sys/fs/cgroup/system.slice/runner.service/cgroup.procs": "", + "run/systemd/system/": None, +} + + +# The inodes of the initial user and PID namespaces (PROC_USER_INIT_INO and PROC_PID_INIT_INO in +# linux/proc_ns.h), written out so that a wrong constant in the collector fails these tests. +INITIAL_USER_NS = 4026531837 +INITIAL_PID_NS = 4026531836 + + +class FakeProbe(collector.Probe): + """A Probe whose namespace inodes are set by the test instead of read from /proc.""" + + def __init__(self, *args, user_inode=INITIAL_USER_NS, pid_inode=INITIAL_PID_NS, **kwargs): + super().__init__(*args, **kwargs) + self.inodes = {"user": user_inode, "pid": pid_inode} + + def namespace_inode(self, name): + if self.inodes[name] is None: + raise FileNotFoundError(name) + return self.inodes[name] + + +def interpreter_patches(system, interpreter): + """Make the interpreter facts the collector reads match the made-up machine, not the host that runs the tests.""" + config = collector.sysconfig.get_config_var + version = f"{sys.version_info[0]}{sys.version_info[1]}" + if system == "Linux": + host_soabi = config("SOABI") or f"cpython-{version}-x86_64-linux-gnu" + soabi = re.sub(r"^([a-z]+-[0-9]+[a-z]*)-[a-z0-9_]+-(linux-[a-z0-9]+)$", rf"\g<1>-{interpreter}-\g<2>", host_soabi) + plat = f"linux-{interpreter}" + elif system == "Windows": + soabi, plat = None, {"x86_64": "win-amd64", "aarch64": "win-arm64"}[interpreter] + elif system == "Darwin": + soabi, plat = f"cpython-{version}-darwin", f"macosx-11.0-{'arm64' if interpreter == 'aarch64' else interpreter}" + else: + soabi, plat = None, f"freebsd-14.1-RELEASE-{'amd64' if interpreter == 'x86_64' else interpreter}" + patches = [ + mock.patch.object(collector.sysconfig, "get_config_var", side_effect=lambda name: soabi if name == "SOABI" else config(name)), + mock.patch.object(collector.sysconfig, "get_platform", return_value=plat), + mock.patch.object(collector.sys.implementation, "_multiarch", f"{interpreter}-linux-gnu" if system == "Linux" else None, create=True), + ] + if system != "Linux": + patches.append(mock.patch.object(collector.os, "confstr", side_effect=ValueError("unrecognized configuration name"), create=True)) + return patches + + +@contextlib.contextmanager +def without(module, name): + """Remove an attribute, such as os.getresuid on macOS, for the duration of a test.""" + saved = getattr(module, name) + delattr(module, name) + try: + yield + finally: + setattr(module, name, saved) + + +class FakeMachine: + """A temporary root directory, an environment and a git directory for one made-up machine. + + `machine` is what uname reports; `interpreter` is the architecture the Python + build targets, which emulation and personality changes do not alter. + """ + + def __init__(self, files=None, remove=(), env=None, machine="x86_64", release="6.17.0-1022-azure", system="Linux", interpreter="x86_64"): + self.directory = tempfile.TemporaryDirectory() + self.root = pathlib.Path(self.directory.name) / "root" + content = dict(RUNNER_FILES if files is None else files) + for name in remove: + content.pop(name) + for name, text in content.items(): + path = self.root / name + if text is None: + path.mkdir(parents=True, exist_ok=True) + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + self.root.mkdir(exist_ok=True) + self.git = pathlib.Path(self.directory.name) / "git" + self.git.mkdir() + (self.git / "HEAD").write_text("0123456789abcdef0123456789abcdef01234567\n", encoding="utf-8") + self.env = {"PATH": ""} if env is None else env + self.patches = [ + mock.patch.object(collector.platform, "machine", return_value=machine), + mock.patch.object(collector.platform, "release", return_value=release), + *interpreter_patches(system, interpreter), + ] + self.system = system + + def __enter__(self): + for patch in self.patches: + patch.start() + return self + + def __exit__(self, *exc): + for patch in self.patches: + patch.stop() + self.directory.cleanup() + + def probe(self, **kwargs): + kwargs.setdefault("probe_node", False) + return FakeProbe(self.root, env=self.env, git_dir=self.git, system=self.system, **kwargs) + + def result(self, **kwargs): + return collector.build_result(self.probe(**kwargs), "linux-x64-gha") + + +def value(fact): + return fact["value"] if fact["provenance"] in ("observed", "inferred", "declared") else None + + +def walk_provenance(node, path=""): + """(JSON pointer, provenance) for every fact in a record.""" + if isinstance(node, dict): + if "provenance" in node: + yield path, node["provenance"] + return + for key, item in node.items(): + yield from walk_provenance(item, f"{path}/{key}") + elif isinstance(node, list): + for index, item in enumerate(node): + yield from walk_provenance(item, f"{path}/{index}") + + +def unescaped_pipes(line): + return len(re.findall(r"(?" + checked = [f"{own}#writable", f"{own}/cgroup.procs#writable", f"{own}/cgroup.subtree_control#writable"] + with mock.patch.object(collector.os, "access", side_effect=lambda path, mode: not str(path).endswith("cgroup.threads")): + self.assertEqual(self.isolation(files)["cgroup"]["delegated"], {"value": True, "provenance": "observed", "evidence": checked}) + with mock.patch.object(collector.os, "access", return_value=True): + self.assertEqual(self.isolation(threaded)["cgroup"]["delegated"]["evidence"], checked + [f"{own}/cgroup.threads#writable"]) + # A cgroup.threads that cannot be looked up is not taken as absent, and the probe does not fail. + real_stat = os.stat + + def stat(path, *args, **kwargs): + if str(path).endswith("cgroup.threads"): + raise PermissionError(13, "Permission denied") + return real_stat(path, *args, **kwargs) + + with mock.patch.object(collector.os, "stat", side_effect=stat), mock.patch.object(collector.os, "access", side_effect=lambda path, mode: not str(path).endswith("cgroup.threads")): + self.assertEqual(self.isolation(files)["cgroup"]["delegated"], {"value": False, "provenance": "observed", "evidence": checked + [f"{own}/cgroup.threads#writable"]}) + # access(2) checks the real user ID, so root in any of the IDs, or a capability, says nothing about delegation. + for label, text in (("root", status(uid=0, cap_eff=FULL)), ("real root", status(real_uid=0)), ("saved root", status(saved_uid=0)), + ("permitted capability", status(cap_prm="0000000000000001"))): + with self.subTest(label), mock.patch.object(collector.os, "access", return_value=True): + self.assertEqual(self.isolation(dict(files, **{"proc/self/status": text}))["cgroup"]["delegated"]["provenance"], "unknown") + # /sys/fs/cgroup/../../../etc is the made-up root's etc, which exists; the '..' check alone refuses it. + escape = dict(files, **{"proc/self/cgroup": "0::/../../../etc\n"}) + with mock.patch.object(collector.os, "access", return_value=True): + self.assertEqual(self.isolation(escape)["cgroup"]["delegated"]["provenance"], "unknown") + + def test_security_modules(self): + selinux = dict(RUNNER_FILES) + selinux.update({ + "sys/fs/selinux/enforce": "1", + "sys/kernel/security/lsm": "lockdown,capability,yama,selinux,bpf", + "proc/self/attr/current": "system_u:system_r:container_t:s0\x00", + "sys/module/apparmor/parameters/enabled": "N\n", + }) + isolation = self.isolation(selinux) + self.assertEqual(value(isolation["selinux"]["mode"]), "enforcing") + self.assertEqual(value(isolation["selinux"]["context"]), "system_u:system_r:container_t:s0") + self.assertIs(value(isolation["apparmor"]["enabled"]), False) + shared = dict(selinux, **{"sys/kernel/security/lsm": "capability,apparmor,selinux"}) + self.assertEqual(self.isolation(shared)["selinux"]["context"]["provenance"], "unknown") + compiled_not_mounted = dict(RUNNER_FILES, **{"proc/filesystems": "nodev\tselinuxfs\n"}) + compiled_not_mounted.pop("sys/kernel/security/lsm") + self.assertEqual(self.isolation(compiled_not_mounted)["selinux"]["mode"]["provenance"], "unknown") + not_compiled = dict(compiled_not_mounted, **{"proc/filesystems": "nodev\tsysfs\n"}) + self.assertEqual(value(self.isolation(not_compiled)["selinux"]["mode"]), "disabled") + absent = dict(RUNNER_FILES, **{"sys/module/loop/parameters/max_part": "0\n"}) + absent.pop("sys/module/apparmor/parameters/enabled") + self.assertEqual((value(self.isolation(absent)["apparmor"]["enabled"]), self.isolation(absent)["apparmor"]["enabled"]["provenance"]), (False, "inferred")) + # A file that is there but cannot be read is not evidence that AppArmor is off. + unreadable = dict(RUNNER_FILES, **{"sys/module/apparmor/parameters/enabled/": None}) + unreadable.pop("sys/module/apparmor/parameters/enabled") + apparmor = self.isolation(unreadable)["apparmor"] + self.assertEqual((apparmor["enabled"]["provenance"], apparmor["profile"]["provenance"]), ("unknown", "unknown")) + # The kernel writes Y or N and a newline; any other value, also a padded or longer one, is not evidence either way. + for text in ("garbage\n", "1\n", "y\n", "\n", " Y \n", "Y", "Y" + " " * 15 + "garbage\n", "N\nY\n"): + with self.subTest(enabled=text): + apparmor = self.isolation(dict(RUNNER_FILES, **{"sys/module/apparmor/parameters/enabled": text}))["apparmor"] + self.assertEqual((apparmor["enabled"]["provenance"], apparmor["profile"]["provenance"], apparmor["enabled"]["reason"]), + ("unknown", "unknown", "/sys/module/apparmor/parameters/enabled is not Y or N and a newline, as the kernel writes it")) + no_label = dict(RUNNER_FILES) + no_label.pop("proc/self/attr/apparmor/current") + no_label["proc/self/attr/current"] = "unconfined\n" + self.assertEqual(self.isolation(no_label)["apparmor"]["profile"]["provenance"], "unknown") + + def test_capabilities(self): + self.assertEqual(collector.decode_capabilities("0000000000000000"), []) + self.assertEqual(len(collector.decode_capabilities(FULL)), 41) + self.assertEqual(collector.decode_capabilities("0000000000003000"), ["cap_net_admin", "cap_net_raw"]) + self.assertEqual(collector.decode_capabilities("0000020000000001"), ["cap_chown", "cap_41"]) + files = dict(RUNNER_FILES, **{"proc/self/status": status(cap_eff="0000000000002000")}) + self.assertEqual(value(self.isolation(files)["capabilities"]["effective"]), ["cap_net_raw"]) + # A permitted capability can be raised again, so the policy reads the permitted set too. + files = dict(RUNNER_FILES, **{"proc/self/status": status(cap_prm="0000000000002000")}) + capabilities = self.isolation(files)["capabilities"] + self.assertEqual((value(capabilities["effective"]), value(capabilities["permitted"])), ([], ["cap_net_raw"])) + with FakeMachine(files) as machine: + self.assertIn("policy-minimal-capabilities", validator.validate_document(machine.result(), SCHEMA).codes()) + + def test_a_process_that_can_regain_root(self): + for label, text in (("real", status(real_uid=0)), ("saved", status(saved_uid=0))): + with self.subTest(label), FakeMachine(dict(RUNNER_FILES, **{"proc/self/status": text})) as machine: + result = machine.result() + identity = result["workload"]["identity"] + self.assertIs(value(identity["superuser"]), False) + self.assertEqual(value(identity[f"{label}_uid"]), 0) + findings = validator.validate_document(result, SCHEMA) + self.assertIn(("policy-app-non-root", f"/workload/identity/{label}_uid"), [(code, path) for _, code, path, _ in findings.errors]) + + def test_missing_files_give_unknown_values_without_errors(self): + with FakeMachine({}) as machine: + result = machine.result(user_inode=None) + self.assertEqual(result["probe_errors"], []) + self.assertEqual(result["stages"]["detect"]["status"], "passed") + identity, isolation = result["workload"]["identity"], result["workload"]["isolation"] + self.assertEqual(identity["uid"]["provenance"], "unknown") + self.assertEqual(identity["user_namespace"]["initial"]["provenance"], "unknown") + self.assertEqual(isolation["seccomp"]["provenance"], "unknown") + self.assertEqual(isolation["cgroup"]["version"]["provenance"], "unknown") + self.assertValidRecord(result, policy=False) + self.assertIn("policy-app-non-root", validator.validate_document(result, SCHEMA).codes()) + + def test_a_probe_failure_is_kept_apart_from_the_other_values(self): + files = dict(RUNNER_FILES, **{"proc/self/uid_map": "0 0\n"}) + with FakeMachine(files) as machine: + result = machine.result() + self.assertEqual(result["probe_errors"], [{"probe": "identity", "error": "ValueError raised by the identity probe"}]) + self.assertEqual(result["stages"]["detect"]["status"], "failed") + self.assertEqual(result["overall"], "failed") + self.assertEqual(result["workload"]["identity"]["uid"]["provenance"], "unknown") + self.assertEqual(value(result["workload"]["isolation"]["seccomp"]), "disabled") + self.assertEqual(kinds(result), ["host", "vm"]) + self.assertValidRecord(result, policy=False) + + +class OtherSystemTests(Case): + # Off Linux the collector does not measure the layer kind, elevation or capabilities, so these rules fail. + UNMEASURED = ["policy-app-non-root", "policy-minimal-capabilities", "policy-minimal-devices", "policy-no-privileged-container"] + + def test_a_system_other_than_linux(self): + with FakeMachine(system="Windows", interpreter="aarch64") as machine: + result = machine.result() + layers = result["environment"]["layers"] + self.assertEqual(len(layers), 1) + self.assertEqual(value(layers[0]["os"]["family"]), "windows") + self.assertEqual(layers[0]["kind"]["provenance"], "unknown") + self.assertEqual(result["workload"]["isolation"]["seccomp"]["provenance"], "unknown") + identity = result["workload"]["identity"] + self.assertEqual((identity["superuser"]["provenance"], identity["real_uid"]["provenance"], identity["saved_uid"]["provenance"]), ("unknown", "not-applicable", "not-applicable")) + self.assertEqual(result["probe_errors"], []) + python = result["workload"]["runtimes"]["python"] + self.assertFact(python["arch"], "aarch64", "observed") + self.assertEqual(python["libc"]["provenance"], "unknown") + self.assertValidRecord(result, policy=False) + findings = validator.validate_document(result, SCHEMA) + self.assertEqual(sorted({code for code in findings.codes() if code[len("policy-"):] in validator.RULE_SCOPE}), self.UNMEASURED) + self.assertIn("policy-engine-rootless", findings.codes("warning")) + + def test_user_ids_off_linux(self): + with FakeMachine(system="FreeBSD") as machine: + result = machine.result() + identity = result["workload"]["identity"] + self.assertEqual([identity[name]["evidence"] for name in ("uid", "real_uid", "saved_uid")], [["python:os.getresuid"]] * 3) + self.assertFact(result["workload"]["runtimes"]["python"]["platform"], "freebsd-14.1-RELEASE-amd64", "observed") + # Under Cygwin or MSYS2 user ID 0 does not name the Windows administrator, so nothing is read. + with FakeMachine(system="CYGWIN_NT-10.0-19045") as machine: + identity = machine.result()["workload"]["identity"] + self.assertEqual({identity[name]["provenance"] for name in ("superuser", "uid", "real_uid", "saved_uid")}, {"unknown"}) + # macOS has no getresuid: execve(2) saved the effective user ID, so the saved one is inferred. + with without(collector.os, "getresuid"), mock.patch.object(collector.os, "geteuid", return_value=501), \ + mock.patch.object(collector.os, "getuid", return_value=501), FakeMachine(system="Darwin", interpreter="aarch64") as machine: + result = machine.result() + identity = result["workload"]["identity"] + self.assertEqual(identity["uid"], {"value": 501, "provenance": "observed", "evidence": ["python:os.geteuid"]}) + self.assertEqual(identity["real_uid"], {"value": 501, "provenance": "observed", "evidence": ["python:os.getuid"]}) + self.assertEqual(identity["saved_uid"], {"value": 501, "provenance": "inferred", "evidence": ["python:os.geteuid"]}) + self.assertFact(identity["superuser"], False, "observed") + self.assertEqual(identity["user_namespace"]["initial"]["provenance"], "unknown") + self.assertValidRecord(result, policy=False) + codes = validator.validate_document(result, SCHEMA).codes() + self.assertNotIn("policy-app-non-root", codes) + self.assertIn("policy-minimal-capabilities", codes) + + +class RunEvidenceTests(Case): + def run_env(self, directory, event, payload): + path = pathlib.Path(directory) / "event.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return { + "PATH": "", "GITHUB_ACTIONS": "true", "GITHUB_EVENT_NAME": event, "GITHUB_EVENT_PATH": str(path), + "GITHUB_SHA": "a" * 40, "GITHUB_REF": "refs/pull/18/merge", "GITHUB_REPOSITORY": "P-R-N-D/Claus-AI", + "GITHUB_SERVER_URL": "https://github.com", "GITHUB_RUN_ID": "123456", "GITHUB_RUN_ATTEMPT": "2", + "GITHUB_WORKFLOW_REF": "P-R-N-D/Claus-AI/.github/workflows/ci.yml@refs/pull/18/merge", "GITHUB_JOB": "compatibility", + "RUNNER_ENVIRONMENT": "github-hosted", "RUNNER_OS": "Linux", "RUNNER_ARCH": "X64", + "ImageOS": "ubuntu24", "ImageVersion": "20261004.327.1", + "GITHUB_TOKEN": "ghs_" + "S" * 36, "ACTIONS_RUNTIME_TOKEN": "runtime-token-sentinel", + "ACTIONS_ID_TOKEN_REQUEST_URL": "https://token.example/sentinel", "GITHUB_ACTOR": "private-actor", + "GITHUB_WORKSPACE": "/home/private-runner/work", "HOME": "/home/private-runner", + } + + def test_a_pull_request_run(self): + payload = { + "pull_request": {"head": {"sha": "b" * 40, "ref": "private-branch", "user": {"login": "private-login"}}, + "base": {"sha": "c" * 40}, "title": "private title", "body": "private body"}, + "sender": {"login": "private-sender"}, + } + with tempfile.TemporaryDirectory() as directory: + env = self.run_env(directory, "pull_request", payload) + with FakeMachine(env=env) as machine: + result = machine.result() + run = result["run"] + self.assertEqual(value(run["ci"]["system"]), "github-actions") + self.assertFact(run["ci"]["system"], "github-actions", "observed") + self.assertFact(run["ci"]["run_url"], "https://github.com/P-R-N-D/Claus-AI/actions/runs/123456", "inferred") + self.assertEqual(value(run["ci"]["run_attempt"]), 2) + self.assertEqual(value(run["revision"]["event_sha"]), "a" * 40) + self.assertEqual(value(run["revision"]["pr_head_sha"]), "b" * 40) + self.assertEqual(value(run["revision"]["pr_base_sha"]), "c" * 40) + self.assertEqual(value(run["revision"]["tested_sha"]), "0123456789abcdef0123456789abcdef01234567") + text = json.dumps(result) + for private in ("ghs_", "runtime-token-sentinel", "token.example", "private-", str(pathlib.Path(directory))): + self.assertNotIn(private, text) + profile = validator.load_json(ROOT / "tests" / "compatibility" / "profiles" / "linux-x64-gha.json") + codes = validator.validate_document(result, SCHEMA, profile).codes() + self.assertNotIn("ci-evidence", codes) + self.assertNotIn("profile-link", codes) + + def test_other_events_and_bad_payloads(self): + with tempfile.TemporaryDirectory() as directory: + push = self.run_env(directory, "push", {"after": "d" * 40, "pusher": {"email": "private@example.com"}}) + with FakeMachine(env=push) as machine: + revision = machine.result()["run"]["revision"] + self.assertEqual(revision["pr_head_sha"]["provenance"], "not-applicable") + for label, payload in (("no pull request", {}), ("bad sha", {"pull_request": {"head": {"sha": "B" * 40}, "base": {"sha": "short"}}})): + with self.subTest(label), FakeMachine(env=self.run_env(directory, "pull_request", payload)) as machine: + result = machine.result() + self.assertEqual(result["run"]["revision"]["pr_head_sha"]["provenance"], "unknown") + self.assertEqual(result["probe_errors"], []) + self.assertIn("ci-evidence", validator.validate_document(result, SCHEMA).codes()) + garbled = self.run_env(directory, "pull_request", {}) + pathlib.Path(garbled["GITHUB_EVENT_PATH"]).write_text("{not json", encoding="utf-8") + with FakeMachine(env=garbled) as machine: + result = machine.result() + self.assertIn("could not be read (JSONDecodeError)", result["run"]["revision"]["pr_head_sha"]["reason"]) + self.assertEqual(value(result["run"]["ci"]["event"]), "pull_request") + self.assertEqual(result["probe_errors"], []) + missing = dict(self.run_env(directory, "pull_request", {}), GITHUB_EVENT_PATH=str(pathlib.Path(directory) / "missing.json")) + with FakeMachine(env=missing) as machine: + result = machine.result() + self.assertIn("could not be read", result["run"]["revision"]["pr_head_sha"]["reason"]) + self.assertEqual(result["probe_errors"], []) + target = {"pull_request": {"head": {"sha": "b" * 40}, "base": {"sha": "c" * 40}}} + with FakeMachine(env=self.run_env(directory, "pull_request_target", target)) as machine: + revision = machine.result()["run"]["revision"] + self.assertEqual((value(revision["pr_head_sha"]), value(revision["pr_base_sha"])), ("b" * 40, "c" * 40)) + no_path = {key: item for key, item in self.run_env(directory, "pull_request", {}).items() if key != "GITHUB_EVENT_PATH"} + with FakeMachine(env=no_path) as machine: + result = machine.result() + self.assertEqual(result["run"]["revision"]["pr_head_sha"], {"value": None, "provenance": "unknown", "reason": "GITHUB_EVENT_PATH is not set"}) + self.assertIn("ci-evidence", validator.validate_document(result, SCHEMA).codes()) + # The validator links a run to its profile by its workflow ref, so a manual dispatch records its ref, its + # workflow ref and its commit for any branch name git allows, within the schema's 200 characters and + # without a character the validator refuses in any value. actions/checkout checks the branch out by its + # name, so .git/HEAD names the branch, whose commit is in a loose ref or in packed-refs. + profile = validator.load_json(ROOT / "tests" / "compatibility" / "profiles" / "linux-x64-gha.json") + base = "P-R-N-D/Claus-AI/.github/workflows/ci.yml@refs/heads/" + sha = "0123456789abcdef0123456789abcdef01234567" + for branch in ("main", "fix/issue#12", "feat/caf\u00e9", "\u4e2d\u6587", "a,b", "it's", "a@b", "a\u3000", "x" * (200 - len(base))): + for packed in (False, True): + env = dict(self.run_env(directory, "workflow_dispatch", {}), GITHUB_REF="refs/heads/" + branch, GITHUB_WORKFLOW_REF=base + branch) + with self.subTest(branch=branch, packed=packed), FakeMachine(env=env) as machine: + (machine.git / "HEAD").write_text(f"ref: refs/heads/{branch}\n", encoding="utf-8") + if packed: + (machine.git / "packed-refs").write_text(f"# pack-refs with: peeled fully-peeled sorted\n{sha} refs/heads/{branch}\n", encoding="utf-8") + else: + loose = machine.git / "refs" / "heads" / branch + loose.parent.mkdir(parents=True, exist_ok=True) + loose.write_text(sha + "\n", encoding="utf-8") + result = machine.result() + run = result["run"] + self.assertEqual((value(run["ci"]["ref"]), value(run["ci"]["workflow_ref"]), value(run["revision"]["tested_sha"])), + ("refs/heads/" + branch, base + branch, sha)) + # An evidence reference is printable ASCII, so a loose ref with another name is cited by HEAD. + evidence = "git:.git/packed-refs" if packed else f"git:.git/refs/heads/{branch}" if branch.isascii() else "git:.git/HEAD" + self.assertEqual(run["revision"]["tested_sha"]["evidence"], [evidence]) + codes = validator.validate_document(result, SCHEMA, profile).codes() + self.assertFalse({"schema", "control-character", "ci-evidence", "profile-link"} & set(codes), codes) + for branch in ("x" * (201 - len(base)), "a b", "a~1", "a:b", "a\\b", "a\x01b", "a\x85b", "a\u202eb", "a\udc80"): + env = dict(self.run_env(directory, "workflow_dispatch", {}), GITHUB_WORKFLOW_REF=base + branch) + with self.subTest(branch=branch), FakeMachine(env=env) as machine: + result = machine.result() + self.assertEqual(result["run"]["ci"]["workflow_ref"]["provenance"], "unknown") + codes = validator.validate_document(result, SCHEMA, profile).codes() + self.assertFalse({"schema", "control-character"} & set(codes), codes) + self.assertTrue({"ci-evidence", "profile-link"} <= set(codes), codes) + # GITHUB_REF is read in the same way. + for ref, recorded in (("refs/" + "x" * 195, True), ("refs/tags/v1.0#rc", True), ("refs/" + "x" * 196, False), ("refs/", False), + ("refs/heads/a b", False), ("refs/heads/a~1", False), ("refs/heads/a\x01", False), ("heads/main", False)): + env = dict(self.run_env(directory, "workflow_dispatch", {}), GITHUB_REF=ref) + with self.subTest(ref=ref), FakeMachine(env=env) as machine: + self.assertEqual(machine.result()["run"]["ci"]["ref"]["provenance"], "observed" if recorded else "unknown") + # Python decodes the environment in the file system encoding, so under another one than UTF-8 a ref that is + # not ASCII is not git's ref; an ASCII one still is. + for branch, encoding, recorded in (("feat/caf\u00e9", "utf-8", True), ("feat/caf\u00e9", "iso8859-1", False), ("main", "iso8859-1", True)): + env = dict(self.run_env(directory, "workflow_dispatch", {}), GITHUB_REF=f"refs/heads/{branch}", + GITHUB_WORKFLOW_REF=f"P-R-N-D/Claus-AI/.github/workflows/ci.yml@refs/heads/{branch}") + with self.subTest(branch=branch, encoding=encoding), FakeMachine(env=env) as machine, \ + mock.patch.object(collector.sys, "getfilesystemencoding", return_value=encoding): + ci = machine.result()["run"]["ci"] + for name, key in (("GITHUB_REF", "ref"), ("GITHUB_WORKFLOW_REF", "workflow_ref")): + expected = {"value": None, "provenance": "unknown", "reason": f"{name} is not ASCII, and the file system encoding is not UTF-8", "evidence": [f"env:{name}"]} + self.assertEqual(ci[key]["provenance"] == "observed", recorded) + if not recorded: + self.assertEqual(ci[key], expected) + odd = dict(push, GITHUB_REPOSITORY="owner/repo|x", RUNNER_ENVIRONMENT="custom", GITHUB_SHA="not-a-sha") + with FakeMachine(env=odd) as machine: + ci = machine.result()["run"] + self.assertEqual(ci["ci"]["repository"]["provenance"], "unknown") + self.assertEqual(ci["ci"]["runner_environment"]["provenance"], "unknown") + self.assertEqual(ci["ci"]["run_url"]["provenance"], "unknown") + self.assertEqual(ci["revision"]["event_sha"]["provenance"], "unknown") + + def test_the_run_attempt_is_read_up_to_the_largest_integer_the_schema_allows(self): + # GitHub sets no upper bound; the schema's integers end at 4294967295, so a larger attempt is not a fact. + with tempfile.TemporaryDirectory() as directory: + for attempt, expected in (("10000", 10000), ("4294967295", 4294967295), ("4294967296", None), ("12345678901", None)): + env = dict(self.run_env(directory, "push", {"after": "d" * 40}), GITHUB_RUN_ATTEMPT=attempt) + with self.subTest(attempt=attempt), FakeMachine(env=env) as machine: + result = machine.result() + self.assertEqual(value(result["run"]["ci"]["run_attempt"]), expected) + self.assertEqual(result["run"]["ci"]["run_attempt"]["provenance"], "unknown" if expected is None else "observed") + codes = validator.validate_document(result, SCHEMA).codes() + self.assertNotIn("schema", codes) + self.assertEqual("ci-evidence" in codes, expected is None) + + def test_outside_github_actions(self): + with FakeMachine(env={"PATH": "", "GITHUB_SHA": "a" * 40}) as machine: + run = machine.result()["run"] + self.assertEqual(value(run["ci"]["system"]), "local") + self.assertEqual(run["revision"]["event_sha"]["provenance"], "not-applicable") + + def test_checkout_commit(self): + sha = "0123456789abcdef0123456789abcdef01234567" + with FakeMachine() as machine: + # A ref that leaves .git must not be followed, even to a file that holds a commit ID. + (pathlib.Path(machine.directory.name) / "outside").write_text(sha + "\n", encoding="utf-8") + for head, files, expected in ( + ("ref: refs/heads/main\n", {"refs/heads/main": sha + "\n"}, sha), + ("ref: refs/heads/main\n", {"packed-refs": f"# pack-refs\n{sha} refs/heads/main\n"}, sha), + ("ref: refs/heads/fix/issue#12\n", {"refs/heads/fix/issue#12": sha + "\n"}, sha), + ("ref: refs/heads/feat/caf\u00e9\n", {"packed-refs": f"{sha} refs/heads/feat/caf\u00e9\n"}, sha), + # Only ASCII blanks end the line: this branch is not "refs/heads/a". + ("ref: refs/heads/a\u3000\n", {"refs/heads/a\u3000": sha + "\n", "refs/heads/a": "f" * 40 + "\n"}, sha), + # Git refuses these refs, and each would reach a file that holds a commit ID. + ("ref: refs/heads/.hidden\n", {"refs/heads/.hidden": sha + "\n"}, None), + ("ref: refs/heads/./main\n", {"refs/heads/main": sha + "\n"}, None), + ("ref: refs//main\n", {"refs/main": sha + "\n"}, None), + ("ref: refs/heads/a..b\n", {"refs/heads/a..b": sha + "\n"}, None), + # Git allows "{", "@" and ".lock" elsewhere in a ref, so these are followed. + ("ref: refs/heads/a{b}@c\n", {"refs/heads/a{b}@c": sha + "\n"}, sha), + ("ref: refs/heads/x.locked/y\n", {"packed-refs": f"{sha} refs/heads/x.locked/y\n"}, sha), + # packed-refs lines end at "\n" only, so U+2028 inside a ref name does not end this line at "refs/heads/b". + ("ref: refs/heads/b\n", {"packed-refs": f"{sha} refs/heads/b\u2028c\n"}, None), + ("ref: refs/heads/main\n", {"packed-refs": f"{sha} refs/heads/main\r\n"}, sha), + ("ref: refs/heads/../../../etc/passwd\n", {}, None), + ("ref: refs/../../outside\n", {}, None), + ("garbage\n", {}, None), + ): + (machine.git / "HEAD").write_text(head, encoding="utf-8") + for name, text in files.items(): + path = machine.git / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + with self.subTest(head=head): + self.assertEqual(value(machine.probe().checkout_sha()), expected) + for name in files: + (machine.git / name).unlink() + # A ref with a character, a length or a form that git or GIT_REF refuses is not followed, even when + # packed-refs lists it. A HEAD that is not UTF-8 is not read as U+FFFD, which names another branch here. + refused = ("refs/heads/a b", "refs/heads/a\u202eb", "refs/heads/a\x01b", "refs/heads/" + "x/" * 95 + "x", + "refs/heads/main.", "refs/heads/x.lock", "refs/heads/x.lock/y", "refs/heads/a@{b}", "refs/heads/a./b") + for ref, listed in [(ref.encode(), ref.encode()) for ref in refused] + [(b"refs/heads/x/a\xff", "refs/heads/x/a\ufffd".encode())]: + (machine.git / "HEAD").write_bytes(b"ref: " + ref + b"\n") + (machine.git / "packed-refs").write_bytes(sha.encode() + b" " + listed + b"\n") + with self.subTest(head=ref): + self.assertEqual(machine.probe().checkout_sha(), {"value": None, "provenance": "unknown", "reason": ".git/HEAD has an unexpected form"}) + # packed-refs is read in the same way: a line that is not UTF-8 is not read as U+FFFD, the name of the + # checked-out branch in the first case, and does not stop the lookup in the lines after it. + other = b"f" * 40 + for ref, packed in ((b"refs/heads/x/a\xef\xbf\xbd", other + b" refs/heads/x/a\x80\n" + sha.encode() + b" refs/heads/x/a\xef\xbf\xbd\n"), + (b"refs/heads/main", other + b" refs/heads/x/\xff\n" + sha.encode() + b" refs/heads/main\n")): + (machine.git / "HEAD").write_bytes(b"ref: " + ref + b"\n") + (machine.git / "packed-refs").write_bytes(packed) + with self.subTest(head=ref): + self.assertEqual(machine.probe().checkout_sha(), {"value": sha, "provenance": "observed", "evidence": ["git:.git/packed-refs"]}) + # Under a file system encoding other than UTF-8, the path of a ref that is not ASCII is not git's name for + # it, so that ref is not followed, not even when packed-refs lists it. + # The loose ref holds a newer commit than packed-refs, so the guard is shown to come before either is read. + (machine.git / "packed-refs").write_text(f"{sha} refs/heads/feat/caf\u00e9\n{sha} refs/heads/main\n", encoding="utf-8") + (machine.git / "refs/heads/feat").mkdir(parents=True, exist_ok=True) + (machine.git / "refs/heads/feat/caf\u00e9").write_text("e" * 40 + "\n", encoding="utf-8") + for branch, encoding, expected in (("feat/caf\u00e9", "utf-8", "e" * 40), ("feat/caf\u00e9", "iso8859-1", None), + ("feat/caf\u00e9", "ascii", None), ("main", "iso8859-1", sha)): + (machine.git / "HEAD").write_text(f"ref: refs/heads/{branch}\n", encoding="utf-8") + with self.subTest(branch=branch, encoding=encoding), mock.patch.object(collector.sys, "getfilesystemencoding", return_value=encoding): + result = machine.probe().checkout_sha() + self.assertEqual(value(result), expected) + if expected is None: + self.assertEqual(result["reason"], "a ref that is not ASCII is followed only when the file system encoding is UTF-8") + (machine.git / "refs/heads/feat/caf\u00e9").unlink() + # Only a loose ref that does not exist is looked up in packed-refs: one that exists but cannot be read, or + # holds no commit ID, such as a symbolic ref, may be newer than the packed commit. + read_regular = collector.read_regular + (machine.git / "HEAD").write_text("ref: refs/heads/feat/caf\u00e9\n", encoding="utf-8") + (machine.git / "packed-refs").write_text(f"{sha} refs/heads/feat/caf\u00e9\n", encoding="utf-8") + for error, expected in ((FileNotFoundError, sha), (NotADirectoryError, sha), (PermissionError, None), (collector.NotRegularFile, None), + (UnicodeEncodeError("ascii", "x", 0, 1, "not encodable"), None)): + + def failing(path, limit, error=error): + if pathlib.Path(path).name not in ("HEAD", "packed-refs"): + raise error + return read_regular(path, limit) + with self.subTest(error=error), mock.patch.object(collector, "read_regular", side_effect=failing): + self.assertEqual(value(machine.probe().checkout_sha()), expected) + (machine.git / "refs/heads/feat/caf\u00e9").write_text("ref: refs/heads/main\n", encoding="utf-8") + self.assertEqual(machine.probe().checkout_sha(), {"value": None, "provenance": "unknown", "reason": ".git/refs/heads/feat/caf\u00e9 does not hold a commit ID"}) + (machine.git / "refs/heads/feat/caf\u00e9").unlink() + + def test_only_allowlisted_variables_are_read(self): + probe = collector.Probe(env={"GITHUB_TOKEN": "x"}) + with self.assertRaises(collector.ProbeError): + probe.env("GITHUB_TOKEN") + for name in collector.PRESENCE_ONLY: + with self.subTest(name), self.assertRaises(collector.ProbeError): + probe.env_value(name, ".*") + # WSL 2 is recognised from the kernel release; the WSL variables are not read. + for name in ("WSL_DISTRO_NAME", "WSL_INTEROP"): + with self.subTest(name), self.assertRaises(collector.ProbeError): + collector.Probe(env={name: "x"}).env(name) + + +class NodeTests(Case): + def fake_node(self, directory, output, status=0, before="", after=""): + script = pathlib.Path(directory) / "node" + leak = pathlib.Path(directory) / "leak" + script.write_text( + "#!/bin/sh\n" + f'if [ -n "${{SENTINEL_SECRET+x}}" ] || [ -n "${{HOME+x}}" ]; then echo leaked > "{leak}"; fi\n' + f"{before}printf '%s' '{output}'\n{after}exit {status}\n", + encoding="utf-8", + ) + script.chmod(0o755) + return leak + + def node(self, output, status=0, reference=("x86_64", "x86"), before="", after=""): + with tempfile.TemporaryDirectory() as directory, FakeMachine(env={"PATH": directory}) as machine: + leak = self.fake_node(directory, output, status, before, after) + with mock.patch.dict(os.environ, {"SENTINEL_SECRET": "x", "HOME": "/home/private"}): + probe = machine.probe(probe_node=True) + facts = probe.guard("node", lambda: probe.node(reference), collector.node_fallback) + self.assertFalse(leak.exists(), "node received the parent environment") + return facts, probe.errors + + @unittest.skipUnless(os.name == "posix" and pathlib.Path("/bin/sh").exists(), "needs /bin/sh") + def test_node_runs_with_an_empty_environment(self): + facts, errors = self.node('{"v":"24.21.0","m":"137","n":"10","a":"x64","p":"linux"}') + self.assertEqual(errors, []) + self.assertEqual((value(facts["version"]), value(facts["modules_abi"]), value(facts["arch"]), value(facts["execution"])), ("24.21.0", "137", "x86_64", "native")) + + @unittest.skipUnless(os.name == "posix" and pathlib.Path("/bin/sh").exists(), "needs /bin/sh") + def test_node_on_little_endian_power(self): + # Node reports ppc64 for both byte orders; the name follows Node's own byte order, not this Python's. + for endianness, arch, byteorder in (("LE", "ppc64le", "big"), ("BE", "ppc64", "little"), (None, "ppc64", "little")): + output = json.dumps({"v": "24.21.0", "m": "137", "n": "10", "a": "ppc64", "p": "linux", "e": endianness}) + with self.subTest(endianness=endianness), mock.patch.object(collector.sys, "byteorder", byteorder): + facts, _ = self.node(output, reference=("ppc64le", None)) + self.assertEqual((value(facts["arch"]), facts["arch"]["provenance"]), (arch, "observed")) + self.assertEqual(value(facts["execution"]), "native" if arch == "ppc64le" else None) + + @unittest.skipUnless(os.name == "posix" and shutil.which("node"), "needs node on PATH") + def test_the_expression_runs_on_a_real_node(self): + # Run as the probe runs it, with an empty environment: every key the probe reads is there. + output = subprocess.run([shutil.which("node"), "-p", collector.NODE_EXPRESSION], env={}, stdin=subprocess.DEVNULL, + capture_output=True, text=True, timeout=30, check=True).stdout + values = json.loads(output) + self.assertEqual(set(values), {"v", "m", "n", "a", "p", "e"}) + self.assertIn(values["e"], ("LE", "BE")) + + @unittest.skipUnless(os.name == "posix" and pathlib.Path("/bin/sh").exists(), "needs /bin/sh") + def test_node_failures(self): + facts, errors = self.node("", status=3) + self.assertEqual((facts["version"]["provenance"], errors), ("unknown", [])) + facts, errors = self.node("not json") + self.assertEqual(errors, [{"probe": "node", "error": "JSONDecodeError raised by the node probe"}]) + facts, errors = self.node('{"v":"24.21.0; rm -rf /","m":"x","a":"x64","p":"linux"}') + self.assertEqual((facts["version"]["provenance"], facts["modules_abi"]["provenance"]), ("unknown", "unknown")) + for napi, platform_name, arch in (("10; x", "linux", "x64"), ("10", "Linux OS", "x64"), ("10", "linux", "x64 && id")): + facts, _ = self.node(f'{{"v":"24.21.0","m":"137","n":"{napi}","a":"{arch}","p":"{platform_name}"}}') + with self.subTest(napi=napi, platform=platform_name, arch=arch): + self.assertIn("unknown", (facts["napi_version"]["provenance"], facts["platform"]["provenance"], facts["arch"]["provenance"])) + + @unittest.skipUnless(os.name == "posix" and pathlib.Path("/bin/sh").exists(), "needs /bin/sh") + def test_node_output_and_time_are_bounded(self): + # Only one byte past the limit is read; node is then stopped, even when it ignores the closed pipe and keeps running. + started = time.monotonic() + facts, errors = self.node("x" * (collector.NODE_OUTPUT_LIMIT + 1), before="trap '' PIPE\n", after="sleep 30\n") + self.assertEqual((facts["version"]["reason"], errors), (f"node wrote more than {collector.NODE_OUTPUT_LIMIT} bytes", [])) + self.assertLess(time.monotonic() - started, 20) + # The shell's own child holds stdout open, so stopping only the shell would leave the read waiting. + started = time.monotonic() + with mock.patch.object(collector, "NODE_TIMEOUT", 0.5): + facts, errors = self.node('{"v":"24.21.0"}', before="sleep 30\n") + self.assertEqual((facts["version"]["reason"], errors), ("node did not finish within 0.5 seconds", [])) + self.assertLess(time.monotonic() - started, 20) + # node closes its output and keeps running: the wait after the read stops it, by the deadline the read had. + timeouts, readings = [], [] + + class Waiting(subprocess.Popen): + def wait(self, timeout=None): + timeouts.append(timeout) + return super().wait(timeout) + + def clock(): + # The collector's clock runs a second ahead after its first reading, the one that sets the deadline. + readings.append(None) + return time.monotonic() + (1.0 if len(readings) > 1 else 0.0) + + started = time.monotonic() + with mock.patch.object(collector, "NODE_TIMEOUT", 2.0), mock.patch.object(collector.subprocess, "Popen", Waiting), mock.patch.object(collector, "time", types.SimpleNamespace(monotonic=clock)): + facts, errors = self.node('{"v":"24.21.0"}', before="sleep 0.5\n", after="exec >&-\nsleep 30\n") + self.assertEqual((facts["version"]["reason"], errors), ("node did not finish within 2.0 seconds", [])) + # The wait gets what is left of the read's deadline, under a second here, never a fresh timeout of about two. + self.assertLess(timeouts[0], 1.0) + self.assertLess(time.monotonic() - started, 20) + + @unittest.skipUnless(os.name == "posix" and pathlib.Path("/bin/sh").exists(), "needs /bin/sh") + def test_a_late_timer_does_not_signal_a_reaped_node(self): + # A timer that fires after node was reaped must not signal its process ID, which may already be in use again. + timers = [] + + class Timer: + def __init__(self, interval, function): + self.function = function + timers.append(self) + + def start(self): + pass + + def cancel(self): + pass + + with mock.patch.object(collector.threading, "Timer", Timer), mock.patch.object(collector.os, "killpg") as killpg: + facts, errors = self.node('{"v":"24.21.0","m":"137","n":"10","a":"x64","p":"linux"}') + self.assertEqual((value(facts["version"]), errors), ("24.21.0", [])) + timers.pop().function() + killpg.assert_not_called() + + # The same after the read fails and node is stopped and reaped on the way out. + class Popen(subprocess.Popen): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + pipe = self.stdout + + class Broken: + def __enter__(self): + return self + + def __exit__(self, *exc): + pipe.close() + + def read(self, size): + raise OSError("read failed") + + self.stdout = Broken() + + with mock.patch.object(collector.threading, "Timer", Timer), mock.patch.object(collector.subprocess, "Popen", Popen): + facts, errors = self.node('{"v":"24.21.0"}', after="sleep 30\n") + self.assertEqual(errors, [{"probe": "node", "error": "OSError raised by the node probe"}]) + with mock.patch.object(collector.os, "killpg") as killpg: + timers.pop().function() + killpg.assert_not_called() + + # The same when the timer fires right after the wait reaped node, before the probe returns. It runs in a + # thread of its own, as a timer does, so a probe that holds the lock there cannot hang this test. + fired, signalled = [], [] + + class Reaping(subprocess.Popen): + def wait(self, timeout=None): + status = super().wait(timeout) + fired.append(threading.Thread(target=timers[-1].function, daemon=True)) + fired[-1].start() + return status + + with mock.patch.object(collector.threading, "Timer", Timer), mock.patch.object(collector.subprocess, "Popen", Reaping), \ + mock.patch.object(collector.os, "killpg", lambda pid, sig: signalled.append(pid)): + facts, errors = self.node('{"v":"24.21.0","m":"137","n":"10","a":"x64","p":"linux"}') + for thread in fired: + thread.join(10) + self.assertEqual((value(facts["version"]), errors, len(fired), signalled), ("24.21.0", [], 1, [])) + self.assertFalse(fired[0].is_alive()) + + @unittest.skipUnless(os.name == "posix" and pathlib.Path("/bin/sh").exists(), "needs /bin/sh") + def test_a_timer_that_fires_as_the_read_ends_holds_back_the_reap(self): + # The timer passes its check just as the read ends. node must not be reaped before its signal is sent. + timers, nodes, entered, at_signal = [], [], threading.Event(), [] + + class Timer: + def __init__(self, interval, function): + self.function = function + timers.append(self) + + def start(self): + pass + + def cancel(self): + pass + + class SlowEvent(threading.Event): + def set(self): + # The timer calls this between its check and its signal; wait there long enough for a reap. + entered.set() + time.sleep(0.3) + super().set() + + class Popen(subprocess.Popen): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + nodes.append(self) + pipe = self.stdout + + class Reader: + def __enter__(self): + return self + + def __exit__(self, *exc): + pipe.close() + + def read(self, size): + output = pipe.read(size) + fire.start() + entered.wait(10) + return output + + self.stdout = Reader() + + fire = threading.Thread(target=lambda: timers[-1].function()) + names = types.SimpleNamespace(Event=SlowEvent, Lock=threading.Lock, Timer=Timer) + with mock.patch.object(collector, "threading", names), mock.patch.object(collector.subprocess, "Popen", Popen), \ + mock.patch.object(collector.os, "killpg", lambda pid, sig: at_signal.append(nodes[-1].returncode)): + facts, errors = self.node('{"v":"24.21.0"}') + fire.join(10) + self.assertEqual(at_signal, [None]) + self.assertEqual((facts["version"]["reason"], errors), (f"node did not finish within {collector.NODE_TIMEOUT} seconds", [])) + + def test_node_gets_system_root_alone_on_windows(self): + # Windows needs SystemRoot to start some programs, so there node gets that variable; elsewhere it gets none. + environments = [] + + def popen(*args, **kwargs): + environments.append(kwargs["env"]) + raise OSError("not started") + + with FakeMachine(env={"PATH": "/x", "SystemRoot": "C:\\Windows"}) as machine: + for name in ("posix", "nt"): + probe = machine.probe(probe_node=True) + with self.subTest(os=name), mock.patch.object(collector, "find_on_path", return_value="/x/node"), \ + mock.patch.object(collector.subprocess, "Popen", popen), mock.patch.object(collector.os, "name", name): + self.assertEqual(probe.node(("x86_64", "x86"))["version"]["reason"], "node could not be run (OSError)") + self.assertEqual(environments, [{}, {"SystemRoot": "C:\\Windows"}]) + + def test_node_absent_or_skipped(self): + with tempfile.TemporaryDirectory() as directory, FakeMachine(env={"PATH": directory}) as machine: + self.assertEqual(machine.probe(probe_node=True).node(("x86_64", "x86"))["version"]["reason"], "node is not on PATH") + self.assertEqual(machine.probe(probe_node=False).node(("x86_64", "x86"))["version"]["reason"], "not probed (--no-node)") + + def test_node_is_looked_for_only_on_the_path_given(self): + # An unset PATH searches the default directories; an empty one finds nothing, as shutil.which reads it. + searched = [] + + def which(name, path=None): + searched.append(path) + return None + + for env in ({}, {"PATH": ""}, {"PATH": "/x"}): + with FakeMachine(env=env) as machine, mock.patch.object(collector.shutil, "which", which): + self.assertEqual(machine.probe(probe_node=True).node(("x86_64", "x86"))["version"]["reason"], "node is not on PATH") + self.assertEqual(searched, [os.defpath, "", "/x"]) + + def test_node_is_looked_for_on_windows_only_as_node_exe_in_the_path_directories(self): + # shutil.which on Windows looks in the current directory before PATH, so a node in the checkout would run, + # and tries each extension PATHEXT lists, so a node.bat or node.com earlier on PATH would run in place of node. + started = [] + + def popen(args, **kwargs): + started.append(args[0]) + raise OSError("not started") + + def which(*args, **kwargs): + raise AssertionError("shutil.which is not used on Windows") + + with tempfile.TemporaryDirectory() as directory: + checkout, scripts, installed, empty = (pathlib.Path(directory, name) for name in ("checkout", "scripts", "bin", "empty")) + for place in (checkout, scripts, installed, empty): + place.mkdir() + for path in (checkout / "node.exe", installed / "node.exe", *(scripts / name for name in ("node", "node.bat", "node.cmd", "node.com"))): + path.write_text("", encoding="utf-8") + # With no PATH the default directories are searched, but not the current directory that Windows lists first; + # a PATH that names the current directory has it searched. An empty entry names no directory. + default = os.curdir + os.pathsep + str(installed) + path = str(scripts) + os.pathsep + str(installed) + cases = ((path, str(installed / "node.exe")), (os.pathsep + path, str(installed / "node.exe")), (str(empty), None), ("", None), + (None, str(installed / "node.exe")), (default, os.path.join(os.curdir, "node.exe"))) + # The lookup does not depend on the CPython version, as shutil.which does. The probe is made before + # os.name is changed, as pathlib reads it. + for version in ((3, 11, 9), (3, 12, 0)): + for search_path, expected in cases: + env = {"PATHEXT": ".COM;.BAT;.CMD;.EXE"} if search_path is None else {"PATH": search_path, "PATHEXT": ".COM;.BAT;.CMD;.EXE"} + with self.subTest(path=search_path, version=version), FakeMachine(env=env) as machine, contextlib.chdir(checkout): + probe = machine.probe(probe_node=True) + started.clear() + with mock.patch.object(collector.shutil, "which", which), mock.patch.object(collector.subprocess, "Popen", popen), \ + mock.patch.object(collector.os, "name", "nt"), mock.patch.object(collector.os, "defpath", default), \ + mock.patch.object(collector.sys, "version_info", version): + reason = probe.node(("x86_64", "x86"))["version"]["reason"] + self.assertEqual((started, reason), ([expected], "node could not be run (OSError)") if expected else ([], "node is not on PATH")) + + +class CommandTests(Case): + def run_main(self, *argv): + with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()) as err: + try: + code = collector.main([str(arg) for arg in argv]) + except SystemExit as exc: + code = exc.code + return code, out.getvalue() + err.getvalue() + + def test_stage_options(self): + with tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / "result.json" + for argv in ( + ["--stage", "detect=passed"], + ["--stage", "deploy=passed"], + ["--stage", "install=ok"], + ["--stage-reason", "install=x"], + ["--stage", "install=failed", "--stage-reason", "install=line\nbreak"], + ["--stage", "install=passed", "--stage-evidence", "install=https://example.com"], + ["--stage", "install=passed", "--stage-escalation", "install=root"], + ["--profile-id", "Bad Id"], + ): + with self.subTest(argv=argv): + self.assertEqual(self.run_main("--output", output, "--no-node", *argv)[0], 2) + for option, value, label in (("--stage-reason", "install=token ghp_" + "A" * 36, "GitHub token"), + ("--stage-evidence", "install=log:AKIA" + "ABCDEFGHIJKLMNOP", "AWS access key ID")): + with self.subTest(option=option): + code, text = self.run_main("--output", output, "--no-node", "--stage", "install=failed", option, value) + self.assertEqual(code, 2) + self.assertIn(f"{option} install: looks like a {label}", text) + self.assertNotIn(value.split("=", 1)[1], text) + self.assertFalse(output.exists()) + code, _ = self.run_main("--output", output, "--no-node", "--profile-id", "linux-x64-gha", + "--stage", "install=passed", "--stage-evidence", "install=log:uv pip sync", + "--stage-escalation", "install=none", "--stage", "performance=not-applicable", + "--stage-reason", "performance=not measured on shared runners") + result = json.loads(output.read_text(encoding="utf-8")) + self.assertIn(code, (0, 1)) + self.assertEqual(result["stages"]["install"], {"status": "passed", "evidence": ["log:uv pip sync"], "privilege": {"escalation": {"value": "none", "provenance": "declared"}}}) + self.assertEqual(result["stages"]["initialize"]["status"], "not-run") + self.assertEqual(result["stages"]["performance"]["status"], "not-applicable") + + def test_a_record_that_looks_like_it_holds_a_secret_is_not_written(self): + with tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / "result.json" + with mock.patch.object(collector.platform, "python_version", return_value="3.12.15-ghp_" + "A" * 36): + code, text = self.run_main("--output", output, "--no-node") + self.assertEqual(code, 2) + self.assertIn("looks like a GitHub token; it was not written", text) + self.assertNotIn("ghp_", text) + self.assertFalse(output.exists()) + + def test_the_secret_shapes_match_the_validator(self): + self.assertEqual([(label, shape.pattern) for label, shape in collector.SECRET_SHAPES], + [(label, shape.pattern) for label, shape in validator.SECRET_SHAPES]) + + # One sample per shape, built from parts so the repository holds no token-shaped literal. + SECRET_SAMPLES = ( + *(("gh" + kind + "_" + "A" * 36) for kind in "opsur"), + "github" + "_pat_" + "A" * 22, + "AK" + "IA" + "ABCDEFGHIJKLMNOP", + "AS" + "IA" + "ABCDEFGHIJKLMNOP", + "-----BEGIN " + "RSA PRIVATE KEY-----", + *(("xo" + "x" + kind + "-" + "A" * 12) for kind in "abposr"), + "ey" + "J" + "A" * 10 + ".ey" + "J" + "B" * 10 + ".", + ) + + def test_every_secret_shape_is_refused(self): + for label, shape in collector.SECRET_SHAPES: + with self.subTest(shape=shape.pattern): + self.assertTrue(any(shape.search(sample) for sample in self.SECRET_SAMPLES), f"no sample for {label}") + record = validator.load_json(ROOT / "tests" / "compatibility" / "fixtures" / "linux-x64-vm.result.json") + with tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / "result.json" + for sample in self.SECRET_SAMPLES: + with self.subTest(sample=sample[:6]): + output.unlink(missing_ok=True) + code, text = self.run_main("--output", output, "--no-node", "--stage", "install=failed", "--stage-reason", f"install=see {sample}") + self.assertEqual(code, 2) + self.assertNotIn(sample, text) + self.assertFalse(output.exists()) + document = json.loads(json.dumps(record)) + document["stages"]["install"]["reason"] = f"see {sample}" + self.assertIn("secret-like-value", validator.validate_document(document, SCHEMA).codes()) + # The check reads the JSON text, where a character before the value may be an escape ending in a letter. + for prefix in ("\n", "\u200e"): + for sample in self.SECRET_SAMPLES: + with self.subTest(prefix=prefix, sample=sample[:6]): + output.unlink(missing_ok=True) + document = json.loads(json.dumps(record)) + document["stages"]["install"]["reason"] = prefix + sample + with mock.patch.object(collector, "build_result", return_value=document): + code, text = self.run_main("--output", output, "--no-node") + self.assertEqual(code, 2) + self.assertNotIn(sample, text) + self.assertFalse(output.exists()) + + def test_a_probe_failure_exits_1_after_writing_the_record(self): + with tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / "result.json" + with mock.patch.object(collector.Probe, "identity", side_effect=ValueError("x")): + code, text = self.run_main("--output", output, "--no-node") + self.assertEqual(code, 1) + self.assertIn("probe error: identity", text) + result = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(result["stages"]["detect"]["status"], "failed") + self.assertEqual(result["probe_errors"][0]["probe"], "identity") + + def test_the_output_name_cannot_forge_output_lines(self): + with tempfile.TemporaryDirectory() as directory: + for name in ("::warning title=forged::x.json", "x##[warning]y.json", "gh" + "p_" + "A" * 36 + ".json"): + with self.subTest(name=name[:6]): + code, text = self.run_main("--output", pathlib.Path(directory) / name, "--no-node") + self.assertIn("not shown: looks like a GitHub token" if "A" * 36 in name else ": detect ", text) + self.assertFalse([line for line in text.splitlines() if line.lstrip().startswith("::") or "##[" in line], text) + self.assertNotIn("A" * 36, text) + # The collector writes its lines as the validator does. + self.assertEqual(collector.CONTROL.pattern, validator.CONTROL.pattern) + for sample in ("::a##[b", "a\nb\u2028c", *self.SECRET_SAMPLES, *(repr("\n" + sample) for sample in self.SECRET_SAMPLES)): + with self.subTest(sample=sample[:6]): + self.assertEqual(collector.plain(sample), validator.plain(sample)) + + def test_a_ref_may_hold_any_character_git_allows_that_the_validator_accepts(self): + # Checked on every code point, so a change to the validator's CONTROL changes what a recorded ref may hold. + every = "".join(map(chr, range(0x110000))) + self.assertEqual(set(re.findall(collector.REF_CHAR, every)), set(validator.CONTROL.sub("", every)) - set(" ~^:?*[\\")) + for pattern in (collector.GIT_REF, collector.WORKFLOW_REF): + self.assertTrue(pattern.endswith("refs/" + collector.REF_CHAR + "+")) + + def test_an_unwritable_output_is_a_usage_error(self): + with tempfile.TemporaryDirectory() as directory: + self.assertEqual(self.run_main("--output", pathlib.Path(directory) / "missing" / "r.json", "--no-node")[0], 2) + + def test_an_unwritable_summary_is_reported_as_the_summary(self): + with tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / "result.json" + code, text = self.run_main("--output", output, "--summary", pathlib.Path(directory) / "missing" / "summary.md", "--no-node") + self.assertEqual(code, 2) + self.assertIn("the record was written, but the summary could not be", text) + self.assertTrue(output.is_file()) + code, text = self.run_main("--output", output, "--no-node") + self.assertNotIn(directory, text) + + def test_a_summary_that_names_the_record_leaves_the_record_as_written(self): + # The summary is appended, so naming the record, by any spelling or link, would make it invalid JSON. + with tempfile.TemporaryDirectory() as directory: + directory = pathlib.Path(directory) + output = directory / "result.json" + output.write_text("{}", encoding="utf-8") + (directory / "linked").hardlink_to(output) + for label, summary in (("same name", output), ("another spelling", directory / ".." / directory.name / "result.json"), ("hard link", directory / "linked")): + with self.subTest(summary=label): + code, text = self.run_main("--output", output, "--summary", summary, "--no-node") + self.assertEqual(code, 2) + self.assertIn("--summary names the same file, so no summary was appended", text) + self.assertEqual(json.loads(output.read_text(encoding="utf-8"))["kind"], "result") + summary = directory / "summary.md" + self.assertIn(self.run_main("--output", output, "--summary", summary, "--no-node")[0], (0, 1)) + self.assertIn("### Compatibility record", summary.read_text(encoding="utf-8")) + + def test_argparse_errors_cannot_forge_output_lines(self): + # argparse repeats an argument it rejects, and the program name, in its usage line too; both are written as every other line is. + token = "ghp_" + "A" * 36 + with tempfile.TemporaryDirectory() as directory, mock.patch.object(sys, "argv", ["::program##[error]forged"]): + output = pathlib.Path(directory) / "result.json" + code, text = self.run_main("--output", output, "--no-node", "--bad=##[error]forged", "--x=a\n::error::forged", f"--worse={token}") + self.assertFalse(output.exists()) + self.assertEqual(code, 2) + self.assertIn("unrecognized arguments", text) + self.assertIn("looks like a GitHub token", text) + self.assertNotIn("##[", text) + self.assertNotIn(token, text) + self.assertFalse([line for line in text.splitlines() if line.lstrip().startswith("::")], text) + self.assertIn("usage:", text) + + def test_the_summary_escapes_table_cells(self): + self.assertEqual(collector.markdown_cell("a|b`c\nd"), "a\\|b\\`c d") + self.assertEqual(collector.markdown_cell("‮"), "\\‮") + self.assertEqual(collector.markdown_cell("[x](https://example.invalid) \\"), "\\[x\\]\\(https://example.invalid\\) \\\\\\") + # A VERSION_ID with Markdown in it passes the printable check, so the summary must escape it. + hostile = 'ID=ubuntu\nVERSION_ID="24.04|**x**](https://example.invalid)`"\n' + with FakeMachine(dict(RUNNER_FILES, **{"etc/os-release": hostile})) as machine: + result = machine.result() + self.assertEqual(value(result["environment"]["layers"][1]["os"]["version"]), "24.04|**x**](https://example.invalid)`") + rows = [line for line in collector.summary_lines(result) if line.startswith("| ")] + self.assertTrue(any("24.04" in row for row in rows)) + for row in rows: + with self.subTest(row=row[:30]): + self.assertEqual(unescaped_pipes(row), 3) + self.assertIsNone(re.search(r"(?= 4: + self.assertNotIn(hostname, text) + home = os.path.expanduser("~") + if home not in ("/", "~") and len(home) > 5: + self.assertNotIn(home, text) + + def test_this_machine_gives_a_valid_record(self): + text, completed = self.collect({}) + result = json.loads(text) + self.assertIs(result["synthetic"], False) + self.assertEqual(completed.returncode, 0 if not result["probe_errors"] else 1) + self.assertValidRecord(result, policy=False) + + +class SharedTableTests(unittest.TestCase): + """The collector and the validator each keep a copy of these tables; the copies must agree.""" + + def test_architecture_names(self): + self.assertEqual(collector.ARCH_ALIASES, validator.ARCH_ALIASES) + # The names the ELF header can give, and which of them are 32-bit, match the validator's pointer widths. + self.assertEqual(collector.KNOWN_ARCHES, set(validator.ARCH_BITS)) + self.assertEqual(collector.ELF_32_BIT, {arch for arch, bits in validator.ARCH_BITS.items() if bits == 32}) + + def test_platform_tags(self): + for plat in ("linux-x86_64", "linux-aarch64", "linux-armv7l", "linux-i686", "win32", "win-amd64", "win-arm64", "win-arm32", + "macosx-11.0-arm64", "macosx-10.9-x86_64", "macosx-10.9-universal2", "freebsd-14.1-RELEASE-amd64", + "freebsd-14.1-RELEASE-arm64", "cygwin-3.5.4-x86_64", "linux"): + with self.subTest(plat=plat): + self.assertEqual(collector.split_platform(plat), validator.python_platform_target(plat)) + for plat, target in (("freebsd-14.1-RELEASE-amd64", ("freebsd", "x86_64")), ("freebsd-14.1-RELEASE-p6-arm64", ("freebsd", "aarch64")), + ("macosx-11.0-arm64", ("macos", "aarch64")), ("win-arm64", ("windows", "aarch64")), ("linux-x86_64", ("linux", "x86_64"))): + with self.subTest(plat=plat): + self.assertEqual(collector.split_platform(plat), target) + + def test_layer_technologies(self): + # Every technology the collector can write is on the kind of layer the validator binds it to. + for technology in {name for _, name in collector.DMI_VENDORS} | {"hyper-v"}: + self.assertEqual(validator.TECHNOLOGY_KINDS.get(technology), "vm", technology) + for technology in {name for name in collector.CONTAINER_TECHNOLOGIES.values() if name} | {"docker", "podman", "apptainer"}: + self.assertEqual(validator.TECHNOLOGY_KINDS.get(technology), "container", technology) + self.assertEqual(validator.TECHNOLOGY_KINDS.get("wsl2"), "wsl2") + + def test_stages_and_the_overall_verdict(self): + self.assertEqual(collector.STAGES, validator.STAGES) + for statuses in itertools.product(collector.STATUSES, repeat=len(collector.STAGES)): + stages = {name: {"status": status} for name, status in zip(collector.STAGES, statuses)} + self.assertEqual(collector.overall_of(stages), validator.overall_of(stages), statuses) + + +class SourceTests(unittest.TestCase): + """The collector's own code may not reach for host names, account names, the network or the whole environment.""" + + tree = ast.parse(SCRIPT.read_text(encoding="utf-8")) + + def names(self): + for node in ast.walk(self.tree): + if isinstance(node, ast.Attribute): + parts = [node.attr] + inner = node.value + while isinstance(inner, ast.Attribute): + parts.append(inner.attr) + inner = inner.value + if isinstance(inner, ast.Name): + parts.append(inner.id) + yield node, ".".join(reversed(parts)) + + def test_no_forbidden_modules(self): + imported = set() + for node in ast.walk(self.tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.add((node.module or "").split(".")[0]) + allowed = {"argparse", "datetime", "json", "os", "pathlib", "platform", "re", "shutil", "signal", "stat", "struct", "subprocess", "sys", "sysconfig", "threading", "time"} + self.assertEqual(imported - allowed, set()) + + def test_no_forbidden_calls(self): + forbidden = {"platform.node", "platform.uname", "os.uname", "os.getlogin", "os.environ.items", "os.environ.keys", + "os.environ.values", "os.environ.copy", "os.system", "os.popen", "os.putenv", "os.setuid", "os.seteuid"} + used = {name for _, name in self.names()} + self.assertEqual(used & forbidden, set()) + environ = [node for node, name in self.names() if name == "os.environ"] + self.assertEqual(len(environ), 1, "os.environ is read only through Probe.env") + + def test_one_subprocess_without_a_shell(self): + calls = [node for node in ast.walk(self.tree) if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) and node.func.value.id == "subprocess"] + self.assertEqual([call.func.attr for call in calls], ["Popen"]) + keywords = {keyword.arg: keyword.value for keyword in calls[0].keywords} + self.assertNotIn("shell", keywords) + # node gets an empty environment, or on Windows SystemRoot alone. + self.assertEqual(ast.unparse(keywords["env"]), "{'SystemRoot': system_root} if system_root else {}") + # stderr is discarded, and stdout is read through the bounded reader in Probe.node. + self.assertEqual(ast.unparse(keywords["stderr"]), "subprocess.DEVNULL") + + def test_files_are_opened_for_reading_except_the_outputs(self): + for node in ast.walk(self.tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "open" and len(node.args) > 1: + mode = node.args[1].value if isinstance(node.args[1], ast.Constant) else None + with self.subTest(line=node.lineno): + self.assertIn(mode, ("rb", "a")) + main = next(node for node in self.tree.body if isinstance(node, ast.FunctionDef) and node.name == "main") + writers = [node for node in ast.walk(self.tree) if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr in ("write_text", "write_bytes", "mkdir", "unlink", "rmdir", "chmod")] + self.assertEqual([node.lineno for node in writers], [node.lineno for node in ast.walk(main) if node in writers]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/validate_compatibility.py b/scripts/ci/validate_compatibility.py new file mode 100644 index 0000000..34e1664 --- /dev/null +++ b/scripts/ci/validate_compatibility.py @@ -0,0 +1,1633 @@ +"""Validate compatibility profiles and run results against the model in docs/CI-MATRIX.md. + +A document is a profile (a declared lane) or a result (one run on one lane, as +scripts/ci/collect_environment.py writes it). Each is checked in two layers: + +1. Structure, from tests/compatibility/schema.json, by a small interpreter of + the JSON Schema keywords in SUPPORTED_KEYWORDS. A schema that uses any other + keyword is rejected before any document is read, so the schema cannot ask + for a check this script does not perform. This is not a general JSON Schema + implementation; see docs/CI-MATRIX.md for where its result can differ. +2. Rules across fields that JSON Schema cannot express: provenance and + evidence, layer nesting, OS, CPU and ABI consistency, stage order, CPU + fallback reported as accelerator success, the privilege policy and its + exceptions, the link between a result and its profile, and support claims + against docs/DEPENDENCY-STRATEGY.md. These run only on a document whose + structure is valid. Each finding has a stable code. + +Standard library only. Usage, from the repository root: + python -I scripts/ci/validate_compatibility.py [--profile PROFILE.json] + [--dependency-strategy docs/DEPENDENCY-STRATEGY.md] [--require-real] + [--no-policy] [--summary FILE] DOCUMENT.json... + +Exit status: 0 when every document is valid (warnings allowed), 1 when any +document is invalid, 2 on a usage error or a file or schema that cannot be used. +""" + +import argparse +import datetime +import hashlib +import json +import math +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[2] +DEFAULT_SCHEMA = ROOT / "tests" / "compatibility" / "schema.json" +MAX_DOCUMENT_BYTES = 2 * 1024 * 1024 + +# Keywords that check an instance, and keywords that only annotate or hold subschemas. +CHECKING_KEYWORDS = frozenset({ + "type", "enum", "const", "properties", "required", "additionalProperties", "propertyNames", + "items", "minItems", "maxItems", "minLength", "maxLength", "pattern", "minimum", "maximum", + "$ref", "if", "then", "else", +}) +ANNOTATION_KEYWORDS = frozenset({"$schema", "$id", "$comment", "$defs", "title", "description"}) +SUPPORTED_KEYWORDS = CHECKING_KEYWORDS | ANNOTATION_KEYWORDS +JSON_TYPES = frozenset({"object", "array", "string", "integer", "number", "boolean", "null"}) +DIALECT = "https://json-schema.org/draft/2020-12/schema" +# Pattern syntax that Python's re and ECMA-262 read differently: an escaped letter other than t, n, r, f, +# v, x or u (shorthand classes and anchors follow Unicode or another definition, and \a, \N and \U exist +# in Python only), a \u escape of a surrogate (a code point in Python, a UTF-16 code unit in ECMA-262), +# (? groups other than (?:, {,n}, possessive quantifiers, and a class that opens with ] or ^]. Only an +# unescaped one counts; inside a character class some of these are portable, and are rejected anyway. +NON_PORTABLE_PATTERN = re.compile(r"(? or newer" or "or older", and a test finds it in the sentence of the lane's +# backend lock cell that names the status. A lane listed here gives a profile a status only where the versions the +# profile declares meet all of that status's bounds. +STATUS_CONDITIONS = { + ("Linux x86-64 glibc", "checked"): (("glibc", (2, 39), "or newer"), ("CPython", (3, 12, 13), "or newer")), + ("macOS Apple Silicon", "statically-resolvable"): (("macOS", (15,), "or newer"),), + ("macOS Apple Silicon", "not-supported"): (("macOS", (14,), "or older"),), +} + +ARCH_ALIASES = { + "amd64": "x86_64", "x64": "x86_64", "arm64": "aarch64", "i386": "x86", "i486": "x86", + "i586": "x86", "i686": "x86", "ia32": "x86", "armv7l": "armv7", "armv8l": "armv7", "arm": "armv7", + "loong64": "loongarch64", "powerpc64le": "ppc64le", +} +ARCH_BITS = { + "x86_64": 64, "aarch64": 64, "riscv64": 64, "ppc64le": 64, "s390x": 64, "loongarch64": 64, + "x86": 32, "armv7": 32, +} +# (inner, outer) architecture pairs that run without emulation: 32-bit code on a 64-bit kernel of the same family. +COMPATIBLE = frozenset({("x86", "x86_64"), ("armv7", "aarch64")}) +KNOWN_OS_FAMILIES = frozenset({"linux", "windows", "macos", "freebsd"}) +# The layer kind each documented technology makes. +TECHNOLOGY_KINDS = { + **dict.fromkeys(("kvm", "qemu", "hyper-v", "vmware", "virtualbox", "xen", "parallels", "apple-virtualization", + "firecracker", "amazon", "google"), "vm"), + "wsl2": "wsl2", + **dict.fromkeys(("docker", "podman", "containerd", "cri-o", "lxc", "systemd-nspawn", "apptainer"), "container"), +} +KNOWN_TECHNOLOGIES = frozenset(TECHNOLOGY_KINDS) +KNOWN_BACKEND_APIS = frozenset({ + "cpu", "cuda", "rocm", "oneapi", "level-zero", "opencl", "vulkan-compute", "directcompute", + "directml", "metal", "mps", "coreml", "qnn", "wasm", "webgpu", "webgl", "webnn", +}) +KNOWN_ORCHESTRATION = frozenset({"none", "kubernetes", "k3s", "openshift", "nomad", "compose", "quadlet", "systemd"}) +KNOWN_SERVICE_MANAGERS = frozenset({"none", "systemd", "launchd", "windows-scm", "openrc"}) +NODE_PLATFORMS = {"linux": "linux", "win32": "windows", "darwin": "macos", "freebsd": "freebsd"} + +# C0 and C1 controls, DEL, the Unicode line and paragraph separators and bidirectional controls, and lone surrogates. +CONTROL = re.compile("[\x00-\x1f\x7f-\x9f\u061c\u200e\u200f\u2028-\u202e\u2066-\u2069\ud800-\udfff]") +# No shape starts with \b: an escape written just before a value (repr's \n, a JSON Pointer's ~1, JSON's \u200e) +# ends in a word character and would hide it. +SECRET_SHAPES = ( + ("GitHub token", re.compile(r"gh[opsur]_[A-Za-z0-9]{30,}")), + ("GitHub token", re.compile(r"github_pat_[A-Za-z0-9_]{20,}")), + ("AWS access key ID", re.compile(r"(AKIA|ASIA)[0-9A-Z]{16}(?![0-9A-Z])")), + ("private key", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")), + ("Slack token", re.compile(r"(?:xox[abposr]|xapp)-[A-Za-z0-9-]{10,}")), + # A JSON Web Token: "eyJ", at least 8 more base64url characters and ".", twice. The run that holds the first + # "eyJ" is matched from its start, so each run is tested once and a long run does not make the search quadratic. + ("JSON Web Token", re.compile(r"(? MAX_DOCUMENT_BYTES: + raise ValueError(f"larger than {MAX_DOCUMENT_BYTES} bytes") + text = data.decode("utf-8") + if text.startswith("\ufeff"): + raise ValueError("starts with a byte order mark") + + def pairs(items): + out = {} + for key, value in items: + if key in out: + raise ValueError(f"duplicate key {key!r}") + out[key] = value + return out + + def constant(name): + raise ValueError(f"non-standard number {name}") + + def real(text): + value = float(text) + if not math.isfinite(value): + raise ValueError(f"number out of range {text}") + return value + + def integer(text): + real(text) + return int(text) + + try: + return json.loads(text, object_pairs_hook=pairs, parse_constant=constant, parse_float=real, parse_int=integer) + except RecursionError: + raise ValueError("nested too deeply") from None + + +# --------------------------------------------------------------------------- +# Structure: a JSON Schema subset + + +def check_schema(schema): + """Reject a schema that uses a keyword or value outside the supported subset.""" + if not isinstance(schema, dict): + raise SchemaError("#: the root schema must be an object") + defs = schema.get("$defs", {}) + stack = [("#", schema)] + while stack: + where, node = stack.pop() + if isinstance(node, bool): + continue + if not isinstance(node, dict): + raise SchemaError(f"{where}: a schema must be an object or a boolean") + unknown = sorted(set(node) - SUPPORTED_KEYWORDS) + if unknown: + raise SchemaError(f"{where}: unsupported keyword(s) {', '.join(unknown)}") + for key in ("$defs", "$schema", "$id"): + if key in node and where != "#": + raise SchemaError(f"{where}: {key} is supported only at the root") + if node.get("$schema", DIALECT) != DIALECT: + raise SchemaError(f"{where}: only the {DIALECT} dialect is supported") + for key in ("$id", "$comment", "title", "description"): + if key in node and not isinstance(node[key], str): + raise SchemaError(f"{where}: {key} must be a string") + if "type" in node: + types = node["type"] if isinstance(node["type"], list) else [node["type"]] + if not types or any(not isinstance(t, str) or t not in JSON_TYPES for t in types): + raise SchemaError(f"{where}: bad type {node['type']!r}") + if "required" in node and not (isinstance(node["required"], list) and all(isinstance(r, str) for r in node["required"])): + raise SchemaError(f"{where}: required must be a list of strings") + if "enum" in node and not isinstance(node["enum"], list): + raise SchemaError(f"{where}: enum must be a list") + for key in ("minItems", "maxItems", "minLength", "maxLength"): + if key in node and not (type(node[key]) is int and node[key] >= 0): + raise SchemaError(f"{where}: {key} must be a non-negative integer") + for key in ("minimum", "maximum"): + if key in node and not (type(node[key]) in (int, float)): + raise SchemaError(f"{where}: {key} must be a number") + if "pattern" in node: + try: + re.compile(node["pattern"]) + except (re.error, TypeError) as exc: + raise SchemaError(f"{where}: bad pattern: {exc}") from None + if NON_PORTABLE_PATTERN.search(node["pattern"]): + raise SchemaError(f"{where}: pattern uses syntax that Python and ECMA-262 read differently") + if "$ref" in node: + match = re.fullmatch(r"#/\$defs/([A-Za-z0-9_]+)", str(node["$ref"])) + if not match or match.group(1) not in defs: + raise SchemaError(f"{where}: unresolvable $ref {node['$ref']!r}") + for key in ("properties", "$defs"): + if key in node: + if not isinstance(node[key], dict): + raise SchemaError(f"{where}: {key} must be an object") + stack.extend((f"{where}/{key}/{name}", sub) for name, sub in node[key].items()) + for key in ("items", "additionalProperties", "propertyNames", "if", "then", "else"): + if key in node: + stack.append((f"{where}/{key}", node[key])) + check_ref_cycles(schema, defs) + + +def check_ref_cycles(schema, defs): + """Reject $ref chains that return to a definition without moving to another value, which would never end.""" + def same_value_refs(node): + """Definitions that `node` applies to the value it is applied to.""" + stack, refs = [node], set() + while stack: + node = stack.pop() + if isinstance(node, dict): + if "$ref" in node: + refs.add(node["$ref"].rsplit("/", 1)[1]) + stack.extend(node[key] for key in SAME_VALUE_KEYWORDS if key in node) + return refs + + edges = {name: same_value_refs(sub) for name, sub in defs.items()} + for start in sorted(same_value_refs(schema) | set(defs)): + path, seen = [start], set() + while path: + name = path.pop() + if name in seen: + continue + seen.add(name) + if start in edges.get(name, ()): + raise SchemaError(f"#/$defs/{start}: $ref leads back to itself without moving to another value") + path.extend(edges.get(name, ())) + + +def json_type(value): + if value is None: + return "null" + if isinstance(value, bool): + return "boolean" + if isinstance(value, int): + return "integer" + if isinstance(value, float): + return "number" + if isinstance(value, str): + return "string" + if isinstance(value, list): + return "array" + return "object" + + +def json_equal(a, b): + if isinstance(a, bool) or isinstance(b, bool): + return isinstance(a, bool) and isinstance(b, bool) and a == b + if isinstance(a, (int, float)) and isinstance(b, (int, float)): + return a == b + if type(a) is not type(b): + return False + if isinstance(a, list): + return len(a) == len(b) and all(json_equal(x, y) for x, y in zip(a, b)) + if isinstance(a, dict): + return a.keys() == b.keys() and all(json_equal(a[k], b[k]) for k in a) + return a == b + + +# A path pointer() builds from "" or from a path it built names at most PATH_KEYS keys, each of at most KEY_LIMIT +# characters once escaped. +KEY_LIMIT = 128 +PATH_KEYS = 16 + + +def pointer(path, key): + """The JSON Pointer of `key` under `path`, as a finding names a place. + + A key longer than KEY_LIMIT characters once escaped is named by its length and the start of its SHA-256, and + a path of more than PATH_KEYS keys keeps the first and last of them around "~...", so neither a long key nor + deep nesting makes the path of every value below it long. An escaped key holds "~" only before "0" or "1", + so neither marker reads as a key, but a shortened path can name more than one place: it names the place for + a reader and is not resolved.""" + text = str(key) + name = text.replace("~", "~0").replace("/", "~1") + if len(name) > KEY_LIMIT: + digest = hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest()[:8] + name = f"~({len(text)} characters, sha256 {digest})" + keys = f"{path}/{name}".split("/")[1:] + if len(keys) > PATH_KEYS: + keys = keys[:PATH_KEYS // 2] + ["~..."] + keys[1 - PATH_KEYS // 2:] + return "/" + "/".join(keys) + + +def calendar_date(text): + """Whether a YYYY-MM-DD string (the shape the schema checks) names a real day.""" + try: + datetime.date.fromisoformat(text) + except ValueError: + return False + return True + + +def calendar_time(text): + """Whether a YYYY-MM-DDTHH:MM:SSZ string (the shape the schema checks) is a valid date and time, with seconds 00 to 59.""" + try: + datetime.datetime.strptime(text, "%Y-%m-%dT%H:%M:%SZ") + except ValueError: + return False + return True + + +def validate_schema(instance, schema, root, path, findings): + """Apply `schema` to `instance`, adding one "schema" finding per failed keyword.""" + if schema is True: + return + if schema is False: + findings.error("schema", path or "/", "no value is allowed here") + return + if "$ref" in schema: + validate_schema(instance, root["$defs"][schema["$ref"].rsplit("/", 1)[1]], root, path, findings) + if "type" in schema: + allowed = schema["type"] if isinstance(schema["type"], list) else [schema["type"]] + actual = json_type(instance) + if actual not in allowed and not (actual == "integer" and "number" in allowed): + findings.error("schema", path or "/", f"type is {actual}, expected {' or '.join(allowed)}") + return + if "enum" in schema and not any(json_equal(instance, option) for option in schema["enum"]): + findings.error("schema", path or "/", f"{instance!r} is not one of {schema['enum']}") + if "const" in schema and not json_equal(instance, schema["const"]): + findings.error("schema", path or "/", f"{instance!r} is not {schema['const']!r}") + if isinstance(instance, str): + if "minLength" in schema and len(instance) < schema["minLength"]: + findings.error("schema", path or "/", f"shorter than {schema['minLength']} characters") + if "maxLength" in schema and len(instance) > schema["maxLength"]: + findings.error("schema", path or "/", f"longer than {schema['maxLength']} characters") + if "pattern" in schema and not re.search(schema["pattern"], instance): + findings.error("schema", path or "/", f"does not match {schema['pattern']}") + if isinstance(instance, (int, float)) and not isinstance(instance, bool): + if "minimum" in schema and instance < schema["minimum"]: + findings.error("schema", path or "/", f"less than {schema['minimum']}") + if "maximum" in schema and instance > schema["maximum"]: + findings.error("schema", path or "/", f"greater than {schema['maximum']}") + if isinstance(instance, list): + if "minItems" in schema and len(instance) < schema["minItems"]: + findings.error("schema", path or "/", f"fewer than {schema['minItems']} items") + if "maxItems" in schema and len(instance) > schema["maxItems"]: + findings.error("schema", path or "/", f"more than {schema['maxItems']} items") + if "items" in schema: + for index, item in enumerate(instance): + validate_schema(item, schema["items"], root, pointer(path, index), findings) + if isinstance(instance, dict): + for name in schema.get("required", ()): + if name not in instance: + findings.error("schema", path or "/", f"missing required property {name!r}") + properties = schema.get("properties", {}) + for name, value in instance.items(): + if "propertyNames" in schema: + validate_schema(name, schema["propertyNames"], root, pointer(path, name), findings) + if name in properties: + validate_schema(value, properties[name], root, pointer(path, name), findings) + elif "additionalProperties" in schema: + if schema["additionalProperties"] is False: + findings.error("schema", pointer(path, name), "property is not allowed") + else: + validate_schema(value, schema["additionalProperties"], root, pointer(path, name), findings) + if "if" in schema: + probe = Findings() + validate_schema(instance, schema["if"], root, path, probe) + branch = "then" if not probe.items else "else" + if branch in schema: + validate_schema(instance, schema[branch], root, path, findings) + + +# --------------------------------------------------------------------------- +# Helpers for rules across fields + + +def walk(document, extensions=True): + """Yield (path, value) for every value in the document, without recursion; optionally skip extension contents. + + A value's path is built when the value is reached, so a container's children share its path until then.""" + stack = [("", None, document)] + while stack: + parent, key, value = stack.pop() + path = parent if key is None else pointer(parent, key) + yield path, value + if isinstance(value, dict): + stack.extend((path, key, item) for key, item in value.items() if extensions or key != "extensions") + elif isinstance(value, list): + stack.extend((path, index, item) for index, item in enumerate(value)) + + +def is_fact(value): + return isinstance(value, dict) and "provenance" in value and "value" in value + + +def known(fact): + """The value of a fact whose provenance is observed, inferred or declared, else None.""" + if is_fact(fact) and fact["provenance"] in KNOWN: + return fact["value"] + return None + + +def measured(fact): + """The value of a fact whose provenance is observed or inferred, else None.""" + if is_fact(fact) and fact["provenance"] in MEASURED: + return fact["value"] + return None + + +def normalize_arch(name): + return ARCH_ALIASES.get(name, name) if isinstance(name, str) else None + + +def arch_family(arch): + if arch in ("x86_64", "x86"): + return "x86" + if arch in ("aarch64", "armv7"): + return "arm" + return None + + +def runs_natively_on(arch, other): + """Whether code built for `arch` can run without emulation where `other` is native; None if either is not listed.""" + if arch not in ARCH_BITS or other not in ARCH_BITS: + return None + return arch == other or (arch, other) in COMPATIBLE + + +def translated_by(arch, enclosing): + """Why code built for `arch` cannot run natively inside the `enclosing` layers (outermost first), or None.""" + for index, layer in enumerate(enclosing): + if known(layer["cpu"]["execution"]) == "emulated": + return f"layer {index} is emulated, so everything inside it is translated" + outer = known(layer["cpu"]["arch"]) + if runs_natively_on(arch, outer) is False: + return f"{arch} code on the {outer} of layer {index} is emulated" + return None + + +def python_platform_target(platform): + """Map sysconfig.get_platform() to (OS family, architecture); either may be None.""" + if not isinstance(platform, str): + return None, None + if platform.startswith("linux-"): + return "linux", normalize_arch(platform[len("linux-"):]) + windows = {"win32": "x86", "win-amd64": "x86_64", "win-arm64": "aarch64", "win-arm32": "armv7"} + if platform in windows: + return "windows", windows[platform] + match = re.fullmatch(r"macosx-[0-9.]+-([A-Za-z0-9_]+)", platform) + if match: + arch = match.group(1) + return "macos", None if arch in ("universal2", "intel", "fat", "fat3", "fat64") else normalize_arch(arch) + match = re.fullmatch(r"freebsd-.+-([A-Za-z0-9_]+)", platform) + if match: + return "freebsd", normalize_arch(match.group(1)) + return None, None + + +def cpython_version(version): + """(major, minor) of a CPython version written as platform.python_version() gives it (3.12.15, 3.13.0rc1, or + 3.14.0a1+ for a build after a release), or as major.minor; None for any other text. No number has a leading zero.""" + number = r"(0|[1-9][0-9]*)" + pattern = rf"{number}\.{number}(?:\.{number}(?:(?:a|b|rc){number})?\+?)?" + match = re.fullmatch(pattern, version) if isinstance(version, str) else None + return match.group(1, 2) if match else None + + +def soabi_target(soabi): + """Map a CPython SOABI to (OS family, architecture); either may be None.""" + if not isinstance(soabi, str): + return None, None + match = re.fullmatch(r"cpython-[0-9]+t?d?-([a-z0-9_]+)-linux-[a-z0-9]+", soabi) + if match: + return "linux", normalize_arch(match.group(1)) + if re.fullmatch(r"cpython-[0-9]+t?d?-darwin", soabi): + return "macos", None + match = re.fullmatch(r"cp[0-9]+t?d?-(win32|win_amd64|win_arm64)", soabi) + if match: + return "windows", {"win32": "x86", "win_amd64": "x86_64", "win_arm64": "aarch64"}[match.group(1)] + return None, None + + +def libc_of_fact(libc): + """The C library a libc fact names: glibc or musl when it starts with that name in any ASCII letter case and + no letter follows, otherwise the fact as written; None for no value.""" + if not isinstance(libc, str): + return None + name = re.match(r"(?ai:glibc|musl)", libc) + # str.isalpha is true for letters only, so a digit, a superscript or a Roman numeral may follow the name. + return name.group(0).lower() if name and not libc[name.end():name.end() + 1].isalpha() else libc + + +def libc_of_tag(soabi): + """The C library a Linux ABI tag names: glibc for gnu (gnueabihf, gnux32), musl (musleabihf), or the tag's + own word, such as android; None for a tag that names none.""" + # The word and the digits after it cannot overlap, so a long tag is searched in linear time. + match = re.search(r"-linux-([a-z]+)(?:[0-9][a-z0-9]*)?$", soabi) if isinstance(soabi, str) else None + if not match: + return None + word = match.group(1) + return "glibc" if word.startswith("gnu") else "musl" if word.startswith("musl") else word + + +def libc_family(python): + """(the C library, the path of the fact it came from), from a Python's libc fact or else its SOABI; (None, None) if neither shows it. + + A C library other than glibc or musl is returned as named, so it contradicts a glibc or musl lane. + """ + base = "/workload/runtimes/python" + libc = libc_of_fact(known(python["libc"])) + if libc: + return libc, f"{base}/libc" + libc = libc_of_tag(known(python["soabi"])) + return (libc, f"{base}/soabi") if libc else (None, None) + + +def layers_of(document): + return document["environment"]["layers"] + + +# --------------------------------------------------------------------------- +# Rules that apply to every document + + +def check_strings(document, findings): + for path, value in walk(document): + texts = [value] if isinstance(value, str) else list(value) if isinstance(value, dict) else [] + for text in texts: + if CONTROL.search(text): + findings.error("control-character", path or "/", "contains a control character, a bidirectional formatting character or a lone surrogate") + for label, shape in SECRET_SHAPES: + if shape.search(text): + findings.error("secret-like-value", path or "/", f"looks like a {label}; records are published as CI artifacts") + + +def check_facts(document, findings): + profile = document["kind"] == "profile" + for path, value in walk(document, extensions=False): + if not is_fact(value): + continue + provenance = value["provenance"] + has_value = value["value"] is not None + evidence = value.get("evidence") or [] + if provenance in ("observed", "inferred"): + if not has_value: + findings.error("fact-provenance", path, f"{provenance} fact has no value; use unknown with a reason") + if not evidence: + findings.error("fact-provenance", path, f"{provenance} fact has no evidence") + elif provenance == "declared": + if not has_value: + findings.error("fact-provenance", path, "declared fact has no value; use unknown with a reason") + else: + if has_value: + findings.error("fact-provenance", path, f"{provenance} fact has a value") + if not value.get("reason"): + findings.error("fact-provenance", path, f"{provenance} fact has no reason") + if profile and provenance in ("observed", "inferred"): + findings.error("profile-provenance", path, "a profile declares; it cannot hold observed or inferred facts") + + +def evidence_refs(document): + for path, value in walk(document, extensions=False): + if isinstance(value, dict) and isinstance(value.get("evidence"), list): + for index, ref in enumerate(value["evidence"]): + yield pointer(pointer(path, "evidence"), index), ref + + +def check_synthetic(document, findings): + synthetic = document["synthetic"] + for path, ref in evidence_refs(document): + if synthetic and not ref.startswith("synthetic:"): + findings.error("synthetic-marking", path, "a synthetic document must mark all evidence synthetic:") + if not synthetic and ref.startswith("synthetic:"): + findings.error("synthetic-marking", path, "synthetic: evidence in a document not marked synthetic") + if document["kind"] == "profile": + prefixed = document["id"].startswith("synthetic-") + if synthetic != prefixed: + findings.error("synthetic-marking", "/id", "a profile id starts with synthetic- exactly when the profile is synthetic") + else: + if synthetic: + for keys, fact in facts_of(document["run"]): + if known(fact) is not None: + findings.error("synthetic-marking", f"/run/{'/'.join(keys)}", "a synthetic result names no run, commit or CI system") + if synthetic and document["generated_at"] is not None: + findings.error("synthetic-marking", "/generated_at", "a synthetic result has no collection time") + if not synthetic and document["generated_at"] is None: + findings.error("synthetic-marking", "/generated_at", "a result that is not synthetic records when it was collected") + if document["generated_at"] is not None and not calendar_time(document["generated_at"]): + findings.error("synthetic-marking", "/generated_at", "the collection time is not a valid date and time (seconds run from 00 to 59)") + + +def check_names(document, findings): + """Warn on identifiers outside the documented lists; reject aliases of known architectures.""" + def arch(path, fact): + value = known(fact) + if value is None: + return + if value in ARCH_ALIASES: + findings.error("arch-name", path, f"write {ARCH_ALIASES[value]!r}, not {value!r}") + elif value not in ARCH_BITS: + findings.warning("unrecognized-value", path, f"architecture {value!r} is not in the documented list; no consistency rule applies to it") + + def listed(path, fact, names, what): + value = known(fact) if is_fact(fact) else fact + if value is not None and value not in names: + findings.warning("unrecognized-value", path, f"{what} {value!r} is not in the documented list") + + for index, layer in enumerate(layers_of(document)): + base = f"/environment/layers/{index}" + arch(f"{base}/cpu/arch", layer["cpu"]["arch"]) + listed(f"{base}/os/family", layer["os"]["family"], KNOWN_OS_FAMILIES, "OS family") + listed(f"{base}/technology", layer["technology"], KNOWN_TECHNOLOGIES, "technology") + listed(f"{base}/service_manager/name", layer["service_manager"]["name"], KNOWN_SERVICE_MANAGERS, "service manager") + if "container" in layer: + listed(f"{base}/container/orchestration/name", layer["container"]["orchestration"]["name"], KNOWN_ORCHESTRATION, "orchestration") + runtimes = document["workload"]["runtimes"] + arch("/workload/runtimes/python/arch", runtimes["python"]["arch"]) + arch("/workload/runtimes/node/arch", runtimes["node"]["arch"]) + for index, backend in enumerate(document["compute"]["backends"]): + listed(f"/compute/backends/{index}/api", backend["api"], KNOWN_BACKEND_APIS, "backend API") + + +def check_layers(document, findings): + layers = layers_of(document) + for index, layer in enumerate(layers): + base = f"/environment/layers/{index}" + kind = known(layer["kind"]) + if kind == "host" and index != 0: + findings.error("layer-structure", f"{base}/kind", "a host can only be the outermost layer") + if kind in ("vm", "wsl2", "container") and index == 0: + findings.error("layer-structure", f"{base}/kind", f"a {kind} needs an enclosing layer; add the host, with unknown facts if they cannot be seen") + if "container" in layer and kind != "container": + findings.error("layer-structure", f"{base}/container", "only a layer whose kind is container has container facts") + technology = known(layer["technology"]) + if kind == "host" and technology is not None: + findings.error("layer-structure", f"{base}/technology", f"the outermost host runs on hardware and names no technology, not {technology!r}") + elif kind == "wsl2" and technology is not None and technology != "wsl2": + findings.error("layer-structure", f"{base}/technology", f"a WSL2 layer is made by wsl2, not {technology!r}") + elif kind is not None and technology in TECHNOLOGY_KINDS and TECHNOLOGY_KINDS[technology] != kind: + findings.error("layer-structure", f"{base}/technology", f"{technology!r} makes a {TECHNOLOGY_KINDS[technology]} layer, not a {kind}") + if kind in ("vm", "wsl2", "container") and layer["technology"]["provenance"] == "not-applicable": + findings.error("layer-structure", f"{base}/technology", f"a {kind} is made by a hypervisor or engine; record it as unknown when it cannot be seen") + if kind == "container" and "container" not in layer: + findings.error("layer-structure", base, "a container layer records its engine mode, privileged state, devices and orchestration") + if kind == "container" and known(layer["os"]["kernel"]) is not None: + findings.error("layer-structure", f"{base}/os/kernel", "a container shares its kernel; record it on the layer that runs the kernel") + if kind == "host" and known(layer["cpu"]["execution"]) == "emulated": + findings.error("cpu-contradiction", f"{base}/cpu/execution", "the outermost host cannot be emulated") + if index == 0: + continue + if known(layer["cpu"]["execution"]) == "native": + conflict = translated_by(known(layer["cpu"]["arch"]), layers[:index]) + if conflict: + findings.error("cpu-contradiction", f"{base}/cpu/execution", f"recorded as native, but {conflict}") + parent = layers[index - 1] + family, parent_family = known(layer["os"]["family"]), known(parent["os"]["family"]) + if kind == "container" and family and parent_family and family != parent_family: + findings.error("os-contradiction", f"{base}/os/family", f"a container shares the {parent_family} kernel of the layer below it, so it cannot be {family}") + if kind == "wsl2" and parent_family and parent_family != "windows": + findings.error("os-contradiction", f"{base}/kind", f"WSL2 runs on Windows, not {parent_family}") + if kind == "wsl2" and family and family != "linux": + findings.error("os-contradiction", f"{base}/os/family", "WSL2 runs a Linux kernel") + + +def check_workload(document, findings): + layers = layers_of(document) + family = known(layers[-1]["os"]["family"]) + runtimes = document["workload"]["runtimes"] + python = runtimes["python"] + base = "/workload/runtimes/python" + + platform_family, platform_arch = python_platform_target(known(python["platform"])) + if platform_family and family and platform_family != family: + findings.error("os-contradiction", f"{base}/platform", f"a {platform_family} Python cannot run on {family} in the innermost layer") + + def platform_disagrees(build_arch): + if platform_family == "linux": + # On Linux the platform tag follows uname, so a 32-bit build on a 64-bit kernel is not a contradiction. + return runs_natively_on(build_arch, platform_arch) is False and runs_natively_on(platform_arch, build_arch) is False + # Windows and macOS tags name the build itself. + return platform_arch in ARCH_BITS and build_arch in ARCH_BITS and platform_arch != build_arch + + python_arch = known(python["arch"]) + if platform_disagrees(python_arch): + findings.error("abi-contradiction", f"{base}/arch", f"the platform tag says {platform_arch}, the arch fact says {python_arch}") + bits = known(python["pointer_bits"]) + if python_arch in ARCH_BITS and bits is not None and bits != ARCH_BITS[python_arch]: + findings.error("abi-contradiction", f"{base}/pointer_bits", f"{python_arch} has {ARCH_BITS[python_arch]}-bit pointers, not {bits}") + + for name, runtime in (("python", python), ("node", runtimes["node"])): + if known(runtime["execution"]) == "native": + conflict = translated_by(known(runtime["arch"]), layers) + if conflict: + findings.error("cpu-contradiction", f"/workload/runtimes/{name}/execution", f"recorded as native, but {conflict}") + + node_platform = known(runtimes["node"]["platform"]) + if node_platform in NODE_PLATFORMS and family and NODE_PLATFORMS[node_platform] != family: + findings.error("os-contradiction", "/workload/runtimes/node/platform", f"a {node_platform} Node.js cannot run on {family}") + + if known(python["implementation"]) == "cpython": + version = known(python["version"]) + parts = cpython_version(version) + # Text after the version would otherwise be ignored here and in the lane's interpreter check. + if version is not None and parts is None: + findings.error("abi-contradiction", f"{base}/version", f"{version!r} is not a CPython version such as 3.12.15, 3.13.0rc1 or 3.12, so the cache tag, the ABI tag and a lane cannot be compared with it") + if parts: + major, minor = parts + cache_tag = known(python["cache_tag"]) + if cache_tag is not None and cache_tag != f"cpython-{major}{minor}": + findings.error("abi-contradiction", f"{base}/cache_tag", f"CPython {major}.{minor} has cache tag cpython-{major}{minor}, not {cache_tag!r}") + soabi = re.match(r"(?:cpython-|cp)(\d)(\d+)(t?)", known(python["soabi"]) or "") + if soabi: + if soabi.group(1) != major or soabi.group(2) != minor: + findings.error("abi-contradiction", f"{base}/soabi", f"the ABI tag is for another version than {version!r}") + free_threaded = known(python["free_threaded_build"]) + if free_threaded is not None and free_threaded != bool(soabi.group(3)): + findings.error("abi-contradiction", f"{base}/free_threaded_build", "the free-threaded flag disagrees with the ABI tag's t suffix") + # The platform part of the ABI tag is checked even when the version is unknown. + soabi_family, soabi_arch = soabi_target(known(python["soabi"])) + if known(python["soabi"]) is not None and soabi_family is None: + findings.warning("unrecognized-value", f"{base}/soabi", f"the CPython ABI tag {known(python['soabi'])!r} has no form this validator reads, so its OS and architecture are not checked") + if soabi_family and family and soabi_family != family: + findings.error("os-contradiction", f"{base}/soabi", f"the ABI tag is for {soabi_family}, the innermost layer is {family}") + # An ABI tag architecture outside the list cannot be a listed build architecture, so it is compared too. + if soabi_arch is not None and python_arch in ARCH_BITS and soabi_arch != python_arch: + findings.error("abi-contradiction", f"{base}/soabi", f"the ABI tag is for {soabi_arch}, the arch fact says {python_arch}") + # The ABI tag, like the platform tag, is compared with the innermost layer's OS family and with the arch fact. When + # the family is unknown, or the arch fact is unknown or not a listed name, the two tags are compared with each other. + if family is None and platform_family and soabi_family and platform_family != soabi_family: + findings.error("os-contradiction", f"{base}/soabi", f"the ABI tag is for {soabi_family}, the platform tag is for {platform_family}") + elif python_arch not in ARCH_BITS and platform_family == soabi_family and platform_disagrees(soabi_arch): + findings.error("abi-contradiction", f"{base}/soabi", f"the ABI tag is for {soabi_arch}, the platform tag says {platform_arch}") + tag_libc, fact_libc = libc_of_tag(known(python["soabi"])), libc_of_fact(known(python["libc"])) + # Two names neither of which is glibc or musl (android and bionic) are not compared. + if tag_libc and fact_libc and tag_libc != fact_libc and {tag_libc, fact_libc} & {"glibc", "musl"}: + names = fact_libc if fact_libc in ("glibc", "musl") else "a C library other than glibc or musl (a fact names glibc or musl only when it starts with that name, in any ASCII letter case, with no letter after it)" + findings.error("abi-contradiction", f"{base}/libc", f"the ABI tag is for {tag_libc}, the libc fact {known(python['libc'])!r} names {names}") + if known(python["free_threaded_build"]) is False and known(python["gil_enabled"]) is False: + findings.error("abi-contradiction", f"{base}/gil_enabled", "a build without free threading always holds the GIL") + # glibc and musl are C libraries for Linux, whatever the ABI tag names. + libc = libc_of_fact(known(python["libc"])) + elsewhere = {family, platform_family, soabi_target(known(python["soabi"]))[0]} & {"windows", "macos", "freebsd"} + if libc in ("glibc", "musl") and elsewhere: + findings.error("abi-contradiction", f"{base}/libc", f"{libc} is a C library for Linux, and this Python is for {' and '.join(sorted(elsewhere))}") + + +def check_compute(document, findings): + backends = document["compute"]["backends"] + ids = [backend["id"] for backend in backends] + for index, backend in enumerate(backends): + if ids.count(backend["id"]) > 1: + findings.error("backend-reference", f"/compute/backends/{index}/id", f"duplicate backend id {backend['id']!r}") + if backend["api"] == "cpu" and backend["kind"] != "cpu": + findings.error("backend-reference", f"/compute/backends/{index}", "the cpu API runs on the cpu backend kind") + + +# --------------------------------------------------------------------------- +# Rules for results + + +def overall_of(stages): + statuses = [stages[name]["status"] for name in STAGES] + if "failed" in statuses: + return "failed" + if "passed" in statuses and all(status in ("passed", "not-applicable") for status in statuses): + return "passed" + return "incomplete" + + +def check_stages(document, findings): + stages = document["stages"] + backends = {backend["id"]: backend for backend in document["compute"]["backends"]} + for name in STAGES: + stage = stages[name] + base = f"/stages/{name}" + status = stage["status"] + if status in RAN and not stage.get("evidence"): + findings.error("stage-fields", f"{base}/evidence", f"a {status} stage cites the log or artifact that shows it") + elif status in RAN and not document["synthetic"]: + kinds = STAGE_EVIDENCE + (("tool:",) if name == "detect" else ()) + if not any(ref.startswith(kinds) for ref in stage["evidence"]): + findings.error("stage-fields", f"{base}/evidence", f"a {status} stage of a real run cites a {' or '.join(kinds)} reference") + if status != "passed" and not stage.get("reason"): + findings.error("stage-fields", f"{base}/reason", f"a {status} stage says why") + if status == "not-run" and stage.get("evidence"): + findings.error("stage-fields", f"{base}/evidence", "a stage that did not run has no result to cite; it may be failed or blocked instead") + if status == "not-applicable" and name not in OPTIONAL_STAGES: + findings.error("stage-fields", f"{base}/status", f"every workload has a {name} stage; record it as not-run or blocked with a reason") + if status in RAN and "privilege" not in stage: + findings.error("stage-fields", f"{base}/privilege", f"a {status} stage records whether it escalated privileges") + for prerequisite in PREREQUISITES[name]: + before = stages[prerequisite]["status"] + if status in RAN and before not in ("passed", "not-applicable"): + findings.error("stage-order", f"{base}/status", f"{name} cannot have {status} while {prerequisite} is {before}; record it as blocked or not-run") + compute = stage.get("compute") + if compute is None: + continue + requested = backends.get(compute["requested"]) + effective = backends.get(compute["effective"]) if compute["effective"] is not None else None + if requested is None: + findings.error("backend-reference", f"{base}/compute/requested", f"no backend {compute['requested']!r} in /compute/backends") + if compute["effective"] is not None and effective is None: + findings.error("backend-reference", f"{base}/compute/effective", f"no backend {compute['effective']!r} in /compute/backends") + if status not in RAN and compute["effective"] is not None: + findings.error("backend-reference", f"{base}/compute/effective", f"a {status} stage used no backend") + if status == "passed" and effective is not None and known(effective["available"]) is not True: + findings.error("backend-reference", f"{base}/compute/effective", "the backend that ran is not recorded as available") + if status == "passed" and requested is not None and requested["kind"] in ("gpu", "npu"): + if effective is None or effective["id"] != requested["id"]: + ran = "nothing" if effective is None else f"{effective['id']} ({effective['kind']})" + findings.error("accelerator-fallback", f"{base}/compute", f"{requested['kind']} backend {requested['id']} was requested but {ran} ran; a fallback is not an accelerator pass") + elif status == "passed" and compute["effective"] is None: + findings.error("backend-reference", f"{base}/compute/effective", "a passed stage with a compute record names the backend that ran") + if document["probe_errors"] and stages["detect"]["status"] == "passed": + findings.error("stage-fields", "/stages/detect/status", "detect cannot pass while probe_errors lists probes that failed") + expected = overall_of(stages) + if document["overall"] != expected: + findings.error("overall-mismatch", "/overall", f"the stages give {expected}, not {document['overall']}") + + +def check_identity(document, findings): + identity = document["workload"]["identity"] + family = known(layers_of(document)[-1]["os"]["family"]) + uid, superuser = known(identity["uid"]), known(identity["superuser"]) + # On a POSIX system the superuser is effective UID 0; Windows names it by its token, so there the two are not compared. + if family in POSIX_FAMILIES and uid is not None and superuser is not None and superuser != (uid == 0): + findings.error("identity-contradiction", "/workload/identity/superuser", f"uid {uid} {'is' if uid == 0 else 'is not'} the superuser (in its user namespace, on Linux)") + initial, uid_map = known(identity["user_namespace"]["initial"]), known(identity["user_namespace"]["uid_map"]) + # A nested namespace can also map 0 0 4294967295, so only one direction is checked. + if initial is True and uid_map is not None and uid_map != [[0, 0, 4294967295]]: + findings.error("identity-contradiction", "/workload/identity/user_namespace/uid_map", "the initial user namespace maps 0 0 4294967295") + + +def check_run(document, findings): + ci, revision = document["run"]["ci"], document["run"]["revision"] + if known(ci["system"]) != "github-actions": + return + # The run and the commits are evidence only when measured; a declared value is a claim the record cannot back. + for name in ("event", "repository", "run_id", "run_attempt", "run_url", "workflow_ref", "job"): + if measured(ci[name]) is None: + findings.error("ci-evidence", f"/run/ci/{name}", "a GitHub Actions record names its run, as observed or inferred") + if measured(ci["run_attempt"]) == 0: + findings.error("ci-evidence", "/run/ci/run_attempt", "GitHub numbers a run's attempts from 1") + repository, run_id, url = measured(ci["repository"]), measured(ci["run_id"]), measured(ci["run_url"]) + if repository and run_id and url: + match = re.fullmatch(rf"https://[A-Za-z0-9.-]+/{re.escape(repository)}/actions/runs/{re.escape(run_id)}(?:/attempts/([1-9][0-9]{{0,9}}))?", url) + if not match: + findings.error("ci-evidence", "/run/ci/run_url", "the run URL does not point at this repository's run") + # A URL without an attempt names the run; one with an attempt names that attempt only. An attempt has at + # most 10 digits, as run_attempt has in the schema, so a longer one names no run and int() never reads it. + elif match.group(1) is not None and measured(ci["run_attempt"]) is not None and int(match.group(1)) != measured(ci["run_attempt"]): + findings.error("ci-evidence", "/run/ci/run_url", "the run URL names another attempt than run_attempt") + if measured(revision["tested_sha"]) is None: + findings.error("ci-evidence", "/run/revision/tested_sha", "a GitHub Actions record names the commit its checkout holds, as observed or inferred") + if measured(revision["event_sha"]) is None: + findings.error("ci-evidence", "/run/revision/event_sha", "a GitHub Actions record keeps GITHUB_SHA, as observed or inferred") + if measured(ci["event"]) in ("pull_request", "pull_request_target"): + for name in ("pr_head_sha", "pr_base_sha"): + if measured(revision[name]) is None: + findings.error("ci-evidence", f"/run/revision/{name}", "a pull request run records the head and base it compared, as observed or inferred") + + +# --------------------------------------------------------------------------- +# Privilege policy + + +def effective_policy(profile, findings): + rules = dict(DEFAULT_POLICY) + exceptions = [] + if profile is None: + return rules, exceptions + for rule, level in profile["policy"].get("rules", {}).items(): + if DEFAULT_POLICY[rule] == "required" and level != "required": + findings.error("policy-weakened", f"/policy/rules/{rule}", f"{rule} is required by default; record an approved exception instead of weakening it") + else: + rules[rule] = level + seen = set() + for index, exception in enumerate(profile["policy"].get("exceptions", [])): + base = f"/policy/exceptions/{index}" + if exception["id"] in seen: + findings.error("policy-exception", f"{base}/id", f"duplicate exception id {exception['id']!r}") + seen.add(exception["id"]) + scope_keys = RULE_SCOPE[exception["rule"]] + for key, values in exception["scope"].items(): + if key not in scope_keys and values: + findings.error("policy-exception", f"{base}/scope/{key}", f"an exception to {exception['rule']} cannot be scoped by {key}") + for key in scope_keys: + if not exception["scope"].get(key): + findings.error("policy-exception", f"{base}/scope", f"an exception to {exception['rule']} names the {key} it covers") + approval = exception["approval"] + if approval["status"] == "approved" and not (approval.get("reference") and approval.get("date")): + findings.error("policy-exception", f"{base}/approval", "an approved exception records where and when it was approved") + if approval.get("date") and not calendar_date(approval["date"]): + findings.error("policy-exception", f"{base}/approval/date", f"{approval['date']!r} is not a calendar date") + exceptions.append(exception) + return rules, exceptions + + +def policy_violations(result): + """Yield (rule, item, path, message, detail) for each thing in a result that does not comply. + + `item` maps each of the rule's RULE_SCOPE keys to this violation's value, so that an exception + can be matched with it. It is None when the state is not measured: no exception covers that. + Only observed and inferred facts show compliance; a declared value in a result is a statement. + Findings that share a rule, a path and a message are reported once, with their details joined. + """ + layers = layers_of(result) + family = measured(layers[-1]["os"]["family"]) + identity = result["workload"]["identity"] + superuser = measured(identity["superuser"]) + if superuser is not False: + state = "runs as the superuser" if superuser else "is not shown to run as a non-root user" + yield "app-non-root", {} if superuser else None, "/workload/identity/superuser", f"the workload {state}", None + for name, label in (("real_uid", "real"), ("saved_uid", "saved")): + fact = identity[name] + if fact["provenance"] == "not-applicable" and family is not None and family not in POSIX_FAMILIES: + continue + uid = measured(fact) + if uid == 0: + yield "app-non-root", {}, f"/workload/identity/{name}", f"the workload can regain root: its {label} user ID is 0", None + elif uid is None: + yield "app-non-root", None, f"/workload/identity/{name}", f"the {label} user ID is not measured, so the workload is not shown to be unable to regain root", None + + # A layer that any fact calls a container is held to the container rules, which only measured facts pass. + containers = [index for index, layer in enumerate(layers) if known(layer["kind"]) == "container"] + if not containers: + innermost = measured(layers[-1]["kind"]) + message = None + if family in (None, "linux"): + # WSL 2 runs each distribution in its own PID namespace inside its VM, so there the namespace tells nothing. + if innermost != "wsl2" and measured(result["workload"]["isolation"]["pid_namespace"]["initial"]) is not True: + path, message = "/workload/isolation/pid_namespace/initial", "no layer is a container, but the workload is not shown to run outside one" + elif innermost not in ("host", "vm"): + # Off Linux no namespace probe applies, so only a measured host or VM as the innermost layer rules a container out. + path, message = f"/environment/layers/{len(layers) - 1}/kind", f"no layer is a container, but on {family} only a measured host or VM kind shows the workload runs outside one" + if message is not None: + yield "engine-rootless", None, path, message, None + yield "no-privileged-container", None, path, message, None + yield "minimal-devices", None, path, message, None + for index in containers: + container = layers[index].get("container", {}) + base = f"/environment/layers/{index}/container" + mode = measured(container.get("engine_mode")) + if mode != "rootless": + yield "engine-rootless", {"layers": index} if mode else None, f"{base}/engine_mode", f"the container engine is {mode or 'not shown to be rootless'}", None + privileged = measured(container.get("privileged")) + if privileged is not False: + yield "no-privileged-container", {"layers": index} if privileged else None, f"{base}/privileged", "the container is privileged" if privileged else "the container is not shown to be unprivileged", None + devices = measured(container.get("devices")) + if devices is None: + yield "minimal-devices", None, f"{base}/devices", "the devices passed to the container are not measured", None + for device in devices or []: + yield "minimal-devices", {"devices": device}, f"{base}/devices", "devices are passed to the container", device + for name in STAGES: + stage = result["stages"][name] + if stage["status"] not in RAN: + continue + # The job that ran a stage states its escalation; the collector cannot see another step's privileges. + escalation = known(stage.get("privilege", {}).get("escalation")) + if escalation != "none": + item = {"stages": name, "escalations": escalation} if escalation else None + yield "no-privilege-escalation", item, f"/stages/{name}/privilege/escalation", f"{name} {'used ' + escalation if escalation else 'is not shown to have used no escalation'}", None + for name in ("effective", "permitted"): + fact = result["workload"]["isolation"]["capabilities"][name] + path = f"/workload/isolation/capabilities/{name}" + # Linux capabilities do not exist on another measured OS family. Windows privileges are not recorded (docs/CI-MATRIX.md). + if fact["provenance"] == "not-applicable" and family is not None and family != "linux": + continue + capabilities = measured(fact) + if capabilities is None: + yield "minimal-capabilities", None, path, f"the {name} capabilities are not measured", None + for capability in capabilities or []: + yield "minimal-capabilities", {"capabilities": capability}, path, f"the workload holds {name} capabilities", capability + + +def covers(exception, rule, item): + """Whether an exception covers one violation: same rule, a known state, and every scope value listed.""" + return exception["rule"] == rule and item is not None and all(item[key] in exception["scope"].get(key, []) for key in RULE_SCOPE[rule]) + + +def check_policy(result, profile, findings): + # Errors in the profile's own policy are reported when the profile is validated. + rules, exceptions = effective_policy(profile, Findings()) + applied, groups = {}, {} + for rule, item, path, message, detail in policy_violations(result): + matching = [exception for exception in exceptions if covers(exception, rule, item)] + approved = [exception for exception in matching if exception["approval"]["status"] == "approved"] + if approved: + what = ", ".join(f"{key} {value}" for key, value in item.items()) or path + for exception in approved: + applied.setdefault(exception["id"], []).append(what) + continue + unapproved = ", ".join(f"{exception['id']} ({exception['approval']['status']})" for exception in matching) + if item is None and any(exception["rule"] == rule for exception in exceptions): + message += "; an exception cannot cover a state that is not measured" + details = groups.setdefault((rule, path, message, unapproved), []) + if detail is not None: + details.append(detail) + for (rule, path, message, unapproved), details in groups.items(): + text = f"{message}: {', '.join(details)}" if details else message + if rules[rule] == "preferred": + findings.warning(f"policy-{rule}", path, text) + elif unapproved: + findings.error("policy-exception-not-approved", path, f"{text}; exception {unapproved} is not approved") + else: + findings.error(f"policy-{rule}", path, f"{text}, and no approved exception covers it") + for index, exception in enumerate(exceptions): + if exception["approval"]["status"] != "approved": + continue + if exception["id"] in applied: + # An approval is a reference this validator cannot verify, so every use of one is shown for review. + approval = exception["approval"] + findings.warning("policy-exception-applied", f"/policy/exceptions/{index}", f"exception {exception['id']} ({exception['rule']}) covers {'; '.join(applied[exception['id']])}; approval {approval.get('reference', 'with no reference')}, {approval.get('date', 'undated')}") + else: + findings.warning("policy-exception-unused", "/policy/exceptions", f"exception {exception['id']} covers nothing in this result") + + +# --------------------------------------------------------------------------- +# Profiles, and results against their profile + + +def check_support(profile, strategy_lanes, findings): + support = profile["support"] + status, lane, synthetic = support["status"], support["lane"], profile["synthetic"] + if status == "checked" and synthetic: + findings.error("support-claim", "/support/status", "a synthetic profile cannot claim a checked lane") + if lane is None: + if status != "not-assessed": + findings.error("support-claim", "/support/lane", f"a {status} claim names its docs/DEPENDENCY-STRATEGY.md lane") + return + target = LANE_TARGETS.get(lane) + if target is None: + findings.error("support-claim", "/support/lane", f"no lane {lane!r} is known; name a row of the docs/DEPENDENCY-STRATEGY.md lane table exactly") + return + matches = lane_matches(profile, target, status, findings) + conditional = {name: bounds for (row, name), bounds in STATUS_CONDITIONS.items() if row == lane} + for bound in conditional.get(status, ()): + met = bound_met(profile, bound) + _, path, written, form = declared_version(profile, bound[0]) + shown = "none" if written is None else repr(written) + if met is None: + findings.error("support-claim", path, f"lane {lane!r} is {status} for {bound_text(bound)}, so a {status} claim declares {form}; this profile declares {shown}") + elif not met: + findings.error("support-claim", path, f"lane {lane!r} is {status} for {bound_text(bound)}; this profile declares {shown}") + # Such a lane covers a profile only where the profile's versions meet all the bounds of one of its statuses. + if conditional and not any(all(bound_met(profile, bound) is True for bound in bounds) for bounds in conditional.values()): + matches = False + if status == "not-assessed" and not synthetic and matches: + findings.error("support-claim", "/support/status", "a real profile whose platform, interpreter and versions are what its docs/DEPENDENCY-STRATEGY.md lane covers copies the lane's status") + if strategy_lanes is None: + if status == "checked": + findings.warning("support-claim-unverified", "/support/status", "checked claim not compared with docs/DEPENDENCY-STRATEGY.md; pass --dependency-strategy") + return + if lane not in strategy_lanes: + findings.error("support-claim", "/support/lane", f"docs/DEPENDENCY-STRATEGY.md has no lane {lane!r}") + elif status != "not-assessed" and status not in strategy_lanes[lane]: + recorded = " or ".join(sorted(strategy_lanes[lane])) or "no status this validator reads" + findings.error("support-claim", "/support/status", f"docs/DEPENDENCY-STRATEGY.md records {lane!r} as {recorded}, not {status}") + + +def lane_matches(profile, target, status, findings): + """Compare a profile with what its lane covers; return whether every axis and the interpreter match. + + A declared value that contradicts the lane is an error whatever the status. A status other than + not-assessed also needs each axis the lane names, and the lock's interpreter, to be declared. + """ + layers = layers_of(profile) + python = profile["workload"]["runtimes"]["python"] + kinds = [known(layer["kind"]) for layer in layers] + libc, libc_path = libc_family(python) + axes = ( + ("os", known(layers[-1]["os"]["family"]), f"/environment/layers/{len(layers) - 1}/os/family", "OS family"), + ("arch", known(python["arch"]), "/workload/runtimes/python/arch", "Python architecture"), + ("libc", libc, libc_path or "/workload/runtimes/python/soabi", "C library"), + ("wsl2", True if "wsl2" in kinds else False if None not in kinds else None, "/environment/layers", "WSL2 layer"), + ) + matches = True + for axis, value, path, what in axes: + if target[axis] is None: + continue + if value is not None and value != target[axis]: + findings.error("support-claim", path, f"the profile's {what} is {value!r}; lane {profile['support']['lane']!r} covers {target[axis]!r}") + matches = False + elif value is None: + matches = False + if status != "not-assessed": + findings.error("support-claim", path, f"a {status} claim declares the {what} its lane covers ({target[axis]!r})") + parts = cpython_version(known(python["version"])) + interpreter = (known(python["implementation"]), ".".join(parts) if parts else None, known(python["free_threaded_build"])) + # A version that cannot be read is named as written. + declared = interpreter[1] if parts else repr(known(python["version"])) + if interpreter != LOCK_INTERPRETER: + matches = False + if status != "not-assessed": + findings.error("support-claim", "/workload/runtimes/python", f"the lane table's statuses are for the backend lock's interpreter, CPython {LOCK_INTERPRETER[1]} with the GIL; this profile declares {interpreter[0]} {declared} (free-threaded build: {interpreter[2]}), so it records not-assessed") + return matches + + +def declared_version(profile, name): + """(the version a profile declares for a STATUS_CONDITIONS name as a tuple of numbers, or None when it declares + none that can be read; the path of its fact; the fact's value as written; the form a bound reads). + + macOS is the innermost layer's OS version; glibc is the Python's libc fact, written as the collector writes it in + any ASCII letter case; CPython is a release version, so a pre-release, a build after a release (3.12.14+) or a + major.minor version gives None. + """ + layers, python = layers_of(profile), profile["workload"]["runtimes"]["python"] + part = r"(?:0|[1-9][0-9]{0,3})" # no leading zero, as cpython_version and the collector write numbers + path, fact, pattern, form = { + "macOS": (f"/environment/layers/{len(layers) - 1}/os/version", layers[-1]["os"]["version"], + rf"({part}(?:\.{part}){{0,2}})", "a macOS version, such as 14.7 or 15.5"), + "glibc": ("/workload/runtimes/python/libc", python["libc"], + rf"(?ai:glibc) ({part}(?:\.{part}){{1,2}})", "a libc fact with the glibc version, such as glibc 2.39"), + "CPython": ("/workload/runtimes/python/version", python["version"], + rf"({part}\.{part}\.{part})", "a CPython release version, such as 3.12.15"), + }[name] + written = known(fact) + match = re.fullmatch(pattern, written) if isinstance(written, str) else None + version = tuple(int(part) for part in match.group(1).split(".")) if match else None + return version, path, written, form + + +def bound_met(profile, bound): + """Whether the version a profile declares meets a bound: True, False, or None when it declares none to compare. + + A version is compared on as many parts as the bound has, so macOS 14.7.1 is 14 or older.""" + name, limit, direction = bound + version = declared_version(profile, name)[0] + if version is None or len(version) < len(limit): + return None + head = version[:len(limit)] + return head >= limit if direction == "or newer" else head <= limit + + +def bound_text(bound): + name, limit, direction = bound + return f"{name} {'.'.join(str(part) for part in limit)} {direction}" + + +def check_accelerator(profile, findings): + name = profile.get("accelerator_backend") + if name is None: + return + backend = {backend["id"]: backend for backend in profile["compute"]["backends"]}.get(name) + if backend is None or backend["kind"] not in ("gpu", "npu"): + findings.error("backend-reference", "/accelerator_backend", f"{name!r} is not a gpu or npu backend in /compute/backends") + + +def check_evidence_sources(profile, findings): + if profile["synthetic"]: + return + if not profile["evidence_sources"]: + findings.error("evidence-source", "/evidence_sources", "a real profile names the workflow job that records it") + for index, source in enumerate(profile["evidence_sources"]): + workflow = ROOT / source["workflow"] + base = f"/evidence_sources/{index}" + text = None + if source["workflow"].startswith(".github/workflows/") and ".." not in source["workflow"]: + # A path the system refuses, such as a name over its length limit, or a file that is not UTF-8, is a + # finding, so the path is printed as every finding is, not in a traceback. + try: + text = workflow.read_text(encoding="utf-8") if workflow.is_file() else None + except (OSError, UnicodeError) as exc: + findings.error("evidence-source", f"{base}/workflow", f"cannot read workflow file {source['workflow']!r} ({type(exc).__name__})") + continue + if text is None: + findings.error("evidence-source", f"{base}/workflow", f"no workflow file {source['workflow']!r}") + continue + if not workflow_has_job(text, source["job"]): + findings.error("evidence-source", f"{base}/job", f"{source['workflow']!r} has no job {source['job']!r}") + + +# An unindented jobs key, plain or quoted, with nothing after its colon but blanks and a comment that follows a blank. +# MULTILINE makes ^ and $ match at "\n" only, as the reader below splits lines; read_text has already turned "\r\n" +# and "\r" into "\n". +JOBS_KEY = re.compile(r"""^(?:jobs|"jobs"|'jobs')[ \t]*:(?:[ \t]+(?:#.*)?)?$""", re.MULTILINE) + + +def workflow_has_job(text, job): + """Whether a workflow's top-level `jobs` mapping has the key `job`. + + The workflow is read as text, not parsed as YAML, one line at a time. Lines end at "\n", "\r\n" or "\r" as in YAML + (check_evidence_sources reads the file with universal newlines, and this splits at "\n" only), not at U+0085, + U+2028 or U+2029. The mapping is the block after an unindented `jobs:` line (or `"jobs":`, `'jobs':` or + `jobs :`, with an optional comment after a blank), up to the next line that does not start with a space (blank + and comment lines aside), and its keys are the lines at the indentation of its first key, plain or quoted, with + a colon followed by a space, a tab or the end of the line. A `jobs` mapping written in flow style names no + job.""" + key = re.compile(rf"""(?:{re.escape(job)}|"{re.escape(job)}"|'{re.escape(job)}')[ \t]*:(?:[ \t]|$)""") + jobs = JOBS_KEY.search(text) + if not jobs: + return False + indent, start = None, jobs.end() + 1 + while start < len(text): + end = text.find("\n", start) + end = len(text) if end < 0 else end + line, start = text[start:end], end + 1 + if not line.strip(" \t") or line.lstrip(" \t").startswith("#"): + continue + if not line.startswith(" "): + return False + width = len(line) - len(line.lstrip(" ")) + indent = width if indent is None else indent + if width == indent and key.match(line, width): + return True + return False + + +def facts_of(node, keys=()): + """(key path, fact) for every fact in an object, outside its extensions.""" + if is_fact(node): + yield keys, node + elif isinstance(node, dict): + for key, value in node.items(): + if key != "extensions": + yield from facts_of(value, keys + (key,)) + + +def dig(mapping, keys): + for key in keys: + if not isinstance(mapping, dict) or key not in mapping: + return None + mapping = mapping[key] + return mapping + + +def check_against_profile(result, profile, findings): + if result["profile_id"] != profile["id"]: + findings.error("profile-link", "/profile_id", f"the result names profile {result['profile_id']!r}, not {profile['id']!r}") + if result["synthetic"] != profile["synthetic"]: + findings.error("profile-link", "/synthetic", "a real result is checked against a real profile, and a synthetic one against a synthetic one") + ci = result["run"]["ci"] + job = known(ci["job"]) + if not profile["synthetic"] and measured(ci["system"]) != "github-actions": + findings.error("profile-link", "/run/ci/system", f"profile {profile['id']} is recorded by GitHub Actions jobs; this record was not shown to come from one") + if known(ci["system"]) == "github-actions": + # GITHUB_WORKFLOW_REF is //@. A run is linked to a profile by the workflow and the + # job it ran, so a record that does not name both its repository and its workflow ref links to no profile. + repository, workflow_ref = known(ci["repository"]), known(ci["workflow_ref"]) + sources = [(source["workflow"], source["job"]) for source in profile["evidence_sources"]] + missing = [name for name, fact in (("repository", repository), ("workflow ref", workflow_ref)) if not isinstance(fact, str)] + if missing: + where = "/run/ci/repository" if missing == ["repository"] else "/run/ci/workflow_ref" + findings.error("profile-link", where, f"a GitHub Actions record is linked to profile {profile['id']} by its repository, workflow ref and job; this one does not name its {' and '.join(missing)}") + elif not workflow_ref.startswith(repository + "/"): + findings.error("profile-link", "/run/ci/workflow_ref", f"the workflow ref {workflow_ref!r} is not in repository {repository!r}") + else: + workflow, _, ref = workflow_ref[len(repository) + 1:].partition("@") + if not workflow or not ref: + findings.error("profile-link", "/run/ci/workflow_ref", f"the workflow ref {workflow_ref!r} does not name a workflow file and a ref") + elif (workflow, job) not in sources: + findings.error("profile-link", "/run/ci/workflow_ref", f"job {job!r} of {workflow!r} is not an evidence source of profile {profile['id']}") + + def compare(path, declared_fact, observed_fact): + # Only a measured value establishes what the profile declares; a declared value in the result does not. + declared, observed = known(declared_fact), measured(observed_fact) + if declared is None: + if is_fact(declared_fact) and declared_fact["provenance"] == "not-applicable" and observed is not None: + findings.error("profile-mismatch", path, f"the profile declares this not applicable, the result has {observed!r}") + return + if observed is None: + findings.error("profile-unverified", path, f"the profile declares {declared!r}, but the run did not measure it; declare only what the evidence-source job can observe") + elif not json_equal(declared, observed): + findings.error("profile-mismatch", path, f"the profile declares {declared!r}, the result has {observed!r}") + + declared_layers, observed_layers = layers_of(profile), layers_of(result) + if len(declared_layers) != len(observed_layers): + findings.error("profile-mismatch", "/environment/layers", f"the profile declares {len(declared_layers)} layers, the result has {len(observed_layers)}") + else: + for index, (declared, observed) in enumerate(zip(declared_layers, observed_layers)): + for keys, fact in facts_of(declared): + compare(f"/environment/layers/{index}/{'/'.join(keys)}", fact, dig(observed, keys)) + for keys, fact in facts_of(profile["workload"]["runtimes"]): + compare(f"/workload/runtimes/{'/'.join(keys)}", fact, dig(result["workload"]["runtimes"], keys)) + observed_backends = {backend["id"]: (index, backend) for index, backend in enumerate(result["compute"]["backends"])} + for backend in profile["compute"]["backends"]: + index, other = observed_backends.get(backend["id"], (None, None)) + if other is None: + findings.error("profile-mismatch", "/compute/backends", f"the profile's backend {backend['id']!r} is missing from the result") + elif (other["kind"], other["api"]) != (backend["kind"], backend["api"]): + findings.error("profile-mismatch", f"/compute/backends/{index}", f"backend {backend['id']!r} has another kind or API than the profile declares") + else: + for keys, fact in facts_of(backend): + compare(f"/compute/backends/{index}/{'/'.join(keys)}", fact, dig(other, keys)) + compare("/compute/accelerator_inventory", profile["compute"]["accelerator_inventory"], result["compute"]["accelerator_inventory"]) + for name, fact in profile.get("runner", {}).items(): + compare(f"/run/ci/{name}", fact, ci[name]) + accelerator = profile.get("accelerator_backend") + for name in profile["required_stages"]: + stage = result["stages"][name] + if stage["status"] != "passed": + findings.error("required-stage", f"/stages/{name}/status", f"profile {profile['id']} requires {name} to pass; it is {stage['status']}") + elif accelerator is not None and name in WORKLOAD_STAGES: + compute = stage.get("compute") or {} + if compute.get("requested") != accelerator or compute.get("effective") != accelerator: + ran = compute.get("effective") or "no recorded backend" + findings.error("required-stage", f"/stages/{name}/compute", f"profile {profile['id']} requires {name} to run on {accelerator}; it ran on {ran}") + + +STATUS_WORDS = ( + ("checked", "Checked"), + ("statically-resolvable", "Statically resolvable"), + ("not-supported", "Not supported|Not binary-installable|Incomplete"), +) + + +def read_strategy_lanes(path): + """Map each lane in the DEPENDENCY-STRATEGY.md lane table to the set of its backend lock statuses. + + A cell names one status first, or one per condition ("macOS 15 or newer: statically + resolvable. macOS 14 or older: not binary-installable"); a cell with neither gives an empty set. + """ + text = pathlib.Path(path).read_text(encoding="utf-8") + section = re.search(r"^## Platform and accelerator lanes\n(.*?)(?=^## |\Z)", text, re.MULTILINE | re.DOTALL) + if not section: + raise ValueError("no 'Platform and accelerator lanes' section") + raw = {} + for line in section.group(1).splitlines(): + cells = [cell.strip() for cell in line.strip().strip("|").split("|")] + if not line.startswith("| ") or len(cells) < 3 or cells[0] in ("Lane",) or set(cells[0]) <= set("-: "): + continue + if cells[0] in raw: + raise ValueError(f"the lane table lists {cells[0]!r} twice") + raw[cells[0]] = cells[2] + if not raw: + raise ValueError("no lanes in the lane table") + + def statuses(cell, depth=0): + same = re.match(r"Same as (.+)$", cell) + if same and depth < 3: + return statuses(raw.get(same.group(1).strip(), ""), depth + 1) + for name, words in STATUS_WORDS: + if re.match(rf"(?:{words})\b", cell): + return frozenset({name}) + return frozenset(name for name, words in STATUS_WORDS if re.search(rf": (?:{words})\b", cell, re.IGNORECASE)) + + return {lane: statuses(cell) for lane, cell in raw.items()} + + +# --------------------------------------------------------------------------- +# Entry points + + +def validate_document(document, schema, profile=None, strategy_lanes=None, evaluate_policy=True, require_real=False): + """Return the Findings for one parsed document.""" + findings = Findings() + validate_schema(document, schema, schema, "", findings) + if findings.errors: + findings.warning("semantic-skipped", "/", "rules across fields were not checked because the structure is invalid") + return findings + if require_real and document["synthetic"]: + findings.error("synthetic-marking", "/synthetic", "a synthetic document where a measured one is required") + check_strings(document, findings) + check_facts(document, findings) + check_synthetic(document, findings) + check_names(document, findings) + check_layers(document, findings) + check_workload(document, findings) + check_compute(document, findings) + if document["kind"] == "profile": + check_support(document, strategy_lanes, findings) + check_accelerator(document, findings) + check_evidence_sources(document, findings) + effective_policy(document, findings) + return findings + check_stages(document, findings) + check_identity(document, findings) + check_run(document, findings) + if profile is not None: + check_against_profile(document, profile, findings) + elif document["profile_id"] is not None: + policy = "; the default policy applies" if evaluate_policy else "" + findings.warning("profile-unchecked", "/profile_id", f"the result names a profile that was not given{policy}") + if evaluate_policy: + check_policy(document, profile, findings) + else: + findings.warning("policy-skipped", "/", "the privilege policy was not evaluated (--no-policy), so this verdict says nothing about it") + return findings + + +def load_schema(path=DEFAULT_SCHEMA): + schema = load_json(path) + check_schema(schema) + return schema + + +def plain(text): + """Text for a terminal line or a summary cell: control characters and lone surrogates become spaces; a value + shaped like a secret, and the rest of the line after it, is replaced by a note naming the shape; "##[", which + the GitHub Actions runner reads as a workflow command anywhere in a line, becomes "## [", and "::", which + starts a workflow command at the start of a line (as a file name can), becomes ": :". + """ + text = CONTROL.sub(" ", str(text)) + for label, shape in SECRET_SHAPES: + match = shape.search(text) + if match: + text = text[:match.start()] + f"[not shown: looks like a {label}]" + return text.replace("##[", "## [").replace("::", ": :") + + +def markdown_cell(text): + """Text for one Markdown table cell: control characters become spaces, and Markdown and HTML syntax is escaped.""" + return re.sub(r"([\\`*_~\[\]()<>!|&])", r"\\\1", plain(text)) + + +class ArgumentParser(argparse.ArgumentParser): + def error(self, message): + # argparse repeats the argument it rejects, so its lines are written as every other line is. + usage = "".join(plain(line) + "\n" for line in self.format_usage().splitlines()) + self.exit(2, usage + plain(f"{self.prog}: error: {message}") + "\n") + + +def same_file(path, other): + """Whether two paths name one existing file, however each is written.""" + try: + return path.samefile(other) + except (OSError, ValueError): + return False + + +def same_place(path, other): + """Whether two paths name one file: one existing file, or one place once resolved, also where no file is yet.""" + if same_file(path, other): + return True + try: + return path.resolve() == other.resolve() + except (OSError, RuntimeError, ValueError): + return False + + +def evidence_workflows(document): + """The workflow files a real profile's evidence sources name, which check_evidence_sources reads. + + A profile counts as real here whenever check_evidence_sources would read its files, also under a schema that + lets `synthetic` be something other than a boolean.""" + if not isinstance(document, dict) or document.get("kind") != "profile" or document.get("synthetic"): + return [] + sources = document.get("evidence_sources") + return [ROOT / source["workflow"] for source in sources if isinstance(source, dict) and isinstance(source.get("workflow"), str)] if isinstance(sources, list) else [] + + +def main(argv=None): + parser = ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + parser.add_argument("documents", nargs="+", type=pathlib.Path, help="profile or result JSON files") + parser.add_argument("--schema", type=pathlib.Path, default=DEFAULT_SCHEMA) + parser.add_argument("--profile", type=pathlib.Path, help="profile that every result document is checked against") + parser.add_argument("--dependency-strategy", type=pathlib.Path, help="compare profile support claims with this file's lane table") + parser.add_argument("--require-real", action="store_true", help="reject synthetic documents") + parser.add_argument("--no-policy", action="store_true", help="skip the privilege policy (for records of machines it is not meant for)") + parser.add_argument("--summary", type=pathlib.Path, help="append a Markdown summary to this file") + args = parser.parse_args(argv) + # The summary is appended to, so it must not be the schema, the profile, the lane table or a document. + if args.summary and any(same_file(args.summary, path) for path in (args.schema, args.profile, args.dependency_strategy, *args.documents) if path): + print(plain(f"error: the summary {args.summary} is the schema, the profile, the lane table or a document of this run"), file=sys.stderr) + return 2 + + try: + schema = load_schema(args.schema) + except (OSError, ValueError, SchemaError, RecursionError, TypeError) as exc: + print(plain(f"error: cannot use schema {args.schema}: {exc}"), file=sys.stderr) + return 2 + + def validate(document, *rest, **options): + # The rules across fields read the shape the default schema requires. Another schema can accept a document + # they cannot read: that is a usage error, not a verdict. With the default schema, however its path is + # written, it would be a bug. + try: + return validate_document(document, schema, *rest, **options) + except (KeyError, IndexError, TypeError, AttributeError) as exc: + if args.schema.resolve() == DEFAULT_SCHEMA.resolve(): + raise + print(plain(f"error: schema {args.schema} accepts a document the rules across fields cannot read ({type(exc).__name__})"), file=sys.stderr) + return None + + strategy_lanes = None + if args.dependency_strategy: + try: + strategy_lanes = read_strategy_lanes(args.dependency_strategy) + except (OSError, ValueError) as exc: + print(plain(f"error: cannot read lanes from {args.dependency_strategy}: {exc}"), file=sys.stderr) + return 2 + profile = None + if args.profile: + try: + profile = load_json(args.profile) + except (OSError, ValueError) as exc: + print(plain(f"error: cannot read profile {args.profile}: {exc}"), file=sys.stderr) + return 2 + profile_findings = validate(profile, strategy_lanes=strategy_lanes) if isinstance(profile, dict) else None + if not isinstance(profile, dict) or profile.get("kind") != "profile" or profile_findings is None or profile_findings.errors: + print(plain(f"error: {args.profile} is not a valid profile"), file=sys.stderr) + for level, code, path, message in (profile_findings.items if profile_findings else []): + print(plain(f" {level.upper()} {code} {path}: {message}"), file=sys.stderr) + return 2 + + workflows = evidence_workflows(profile) + invalid = 0 + lines = ["### Compatibility record validation", "", "| Document | Kind | Result | Errors | Warnings |", "|---|---|---|---|---|"] + details = [] + for path in args.documents: + try: + document = load_json(path) + except OSError as exc: + print(plain(f"error: cannot read {path}: {exc.strerror or exc}"), file=sys.stderr) + return 2 + except (ValueError, UnicodeDecodeError) as exc: + findings = Findings() + findings.error("json", "/", f"not strict JSON: {exc}") + document = None + else: + findings = validate(document, profile, strategy_lanes, not args.no_policy, args.require_real) + if findings is None: + return 2 + workflows += evidence_workflows(document) + errors = findings.errors + warnings = [item for item in findings.items if item[0] == "warning"] + verdict = "invalid" if errors else "valid" + invalid += bool(errors) + kind = document.get("kind") if isinstance(document, dict) else "?" + print(plain(f"{path}: {verdict} {kind} ({len(errors)} errors, {len(warnings)} warnings)")) + for level, code, where, message in findings.items: + print(plain(f" {level.upper()} {code} {where}: {message}")) + details.append(f"- {level} `{markdown_cell(code)}` {markdown_cell(path.name)} {markdown_cell(where)}: {markdown_cell(message)}") + lines.append(f"| {markdown_cell(path.name)} | {markdown_cell(kind)} | {verdict} | {len(errors)} | {len(warnings)} |") + + if args.summary: + # Nor may it be a workflow file a real profile's evidence sources were read from, or name one that is missing. + if any(same_place(args.summary, workflow) for workflow in workflows): + print(plain(f"error: the summary {args.summary} is a workflow file a profile of this run names as its evidence source"), file=sys.stderr) + return 2 + shown = details[:200] + if len(details) > len(shown): + shown.append(f"- {len(details) - len(shown)} more findings are in the job log") + lines += [""] + shown + [ + "", + "A valid record shows that the schema, the validator and the probe worked on this runner for this commit. It is not a support claim for this or any other platform; support status is recorded only in docs/DEPENDENCY-STRATEGY.md.", + "", + ] + try: + with open(args.summary, "a", encoding="utf-8") as handle: + handle.write("\n".join(lines)) + except OSError as exc: + print(plain(f"error: cannot write the summary {args.summary}: {exc.strerror or exc}"), file=sys.stderr) + return 2 + return 1 if invalid else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/compatibility/fixtures/kubernetes-arm64-cuda.profile.json b/tests/compatibility/fixtures/kubernetes-arm64-cuda.profile.json new file mode 100644 index 0000000..e2321ef --- /dev/null +++ b/tests/compatibility/fixtures/kubernetes-arm64-cuda.profile.json @@ -0,0 +1,326 @@ +{ + "schema_version": 1, + "kind": "profile", + "synthetic": true, + "id": "synthetic-kubernetes-arm64-cuda", + "title": "Synthetic Kubernetes pod on an aarch64 node with CUDA", + "description": "A non-root pod with one NVIDIA GPU; the GPU device is allowed by an approved exception. A mixed x86-64 and aarch64 cluster has one profile per node architecture.", + "support": { + "status": "not-assessed", + "lane": "Linux aarch64 glibc", + "reference": "docs/DEPENDENCY-STRATEGY.md#platform-and-accelerator-lanes" + }, + "evidence_sources": [], + "required_stages": [ + "detect", + "install", + "initialize", + "execute", + "correctness" + ], + "accelerator_backend": "cuda", + "environment": { + "layers": [ + { + "label": "aarch64 GPU node", + "kind": { + "value": "host", + "provenance": "declared" + }, + "technology": { + "value": null, + "provenance": "not-applicable", + "reason": "the outermost host runs on hardware" + }, + "os": { + "family": { + "value": "linux", + "provenance": "declared" + }, + "name": { + "value": "ubuntu", + "provenance": "declared" + }, + "version": { + "value": "24.04", + "provenance": "declared" + }, + "kernel": { + "value": "6.11.0-1016-nvidia", + "provenance": "declared" + } + }, + "cpu": { + "arch": { + "value": "aarch64", + "provenance": "declared" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + }, + "service_manager": { + "name": { + "value": "systemd", + "provenance": "declared" + }, + "scope": { + "value": "system", + "provenance": "declared" + } + } + }, + { + "label": "Kubernetes pod", + "kind": { + "value": "container", + "provenance": "declared" + }, + "technology": { + "value": "containerd", + "provenance": "declared" + }, + "os": { + "family": { + "value": "linux", + "provenance": "declared" + }, + "name": { + "value": "ubuntu", + "provenance": "declared" + }, + "version": { + "value": "24.04", + "provenance": "declared" + }, + "kernel": { + "value": null, + "provenance": "not-applicable", + "reason": "a container shares the kernel of the layer below" + } + }, + "cpu": { + "arch": { + "value": "aarch64", + "provenance": "declared" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "systemd is not running as the service manager here; other managers are not probed" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "no service manager seen" + } + }, + "container": { + "engine_mode": { + "value": "rootful", + "provenance": "declared" + }, + "privileged": { + "value": false, + "provenance": "declared" + }, + "devices": { + "value": [ + "nvidia-gpu" + ], + "provenance": "declared" + }, + "orchestration": { + "name": { + "value": "kubernetes", + "provenance": "declared" + }, + "version": { + "value": "1.33", + "provenance": "declared" + }, + "scope": { + "value": null, + "provenance": "not-applicable", + "reason": "Kubernetes has no user or system scope" + } + } + } + } + ] + }, + "workload": { + "runtimes": { + "python": { + "implementation": { + "value": "cpython", + "provenance": "declared" + }, + "version": { + "value": "3.12.15", + "provenance": "declared" + }, + "cache_tag": { + "value": "cpython-312", + "provenance": "declared" + }, + "soabi": { + "value": "cpython-312-aarch64-linux-gnu", + "provenance": "declared" + }, + "platform": { + "value": "linux-aarch64", + "provenance": "declared" + }, + "arch": { + "value": "aarch64", + "provenance": "declared" + }, + "pointer_bits": { + "value": 64, + "provenance": "declared" + }, + "free_threaded_build": { + "value": false, + "provenance": "declared" + }, + "gil_enabled": { + "value": true, + "provenance": "declared" + }, + "libc": { + "value": null, + "provenance": "unknown", + "reason": "each run records it" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + }, + "node": { + "version": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "modules_abi": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "napi_version": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "platform": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "arch": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + } + } + } + }, + "compute": { + "backends": [ + { + "id": "cpu", + "kind": "cpu", + "api": "cpu", + "available": { + "value": true, + "provenance": "declared" + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "the CPU model is not recorded" + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "a CPU has no backend version" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "CPU features are not recorded" + } + }, + { + "id": "cuda", + "kind": "gpu", + "api": "cuda", + "available": { + "value": true, + "provenance": "declared" + }, + "device": { + "value": "NVIDIA GB10", + "provenance": "declared" + }, + "version": { + "value": "13.0", + "provenance": "declared" + }, + "features": { + "value": [ + "bf16", + "fp16" + ], + "provenance": "declared" + }, + "extensions": { + "x-driver": "synthetic example of vendor data the validator ignores" + } + } + ], + "accelerator_inventory": { + "value": true, + "provenance": "declared" + } + }, + "policy": { + "rules": {}, + "exceptions": [ + { + "id": "gpu-device", + "rule": "minimal-devices", + "reason": "The CUDA lane needs the GPU device inside the pod.", + "scope": { + "devices": [ + "nvidia-gpu" + ] + }, + "verification": "The device plugin passes exactly one GPU and no other device; the pod spec is reviewed with the lane.", + "end_condition": "Remove when the lane stops running GPU stages.", + "approval": { + "status": "approved", + "reference": "https://github.com/P-R-N-D/Claus-AI (synthetic example, no real approval)", + "date": "2026-10-11" + } + } + ] + }, + "extensions": { + "x-note": "Synthetic fixture for the validator's tests. It describes no machine that was measured." + } +} diff --git a/tests/compatibility/fixtures/kubernetes-arm64-cuda.result.json b/tests/compatibility/fixtures/kubernetes-arm64-cuda.result.json new file mode 100644 index 0000000..c9e5c85 --- /dev/null +++ b/tests/compatibility/fixtures/kubernetes-arm64-cuda.result.json @@ -0,0 +1,750 @@ +{ + "schema_version": 1, + "kind": "result", + "synthetic": true, + "profile_id": "synthetic-kubernetes-arm64-cuda", + "generated_at": null, + "collector": { + "name": "collect-environment", + "version": 1 + }, + "run": { + "ci": { + "system": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "event": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "repository": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "ref": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_id": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_attempt": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_url": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "workflow_ref": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "job": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_environment": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_os": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_arch": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "image_os": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "image_version": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + } + }, + "revision": { + "tested_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "event_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "pr_head_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "pr_base_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + } + } + }, + "environment": { + "layers": [ + { + "label": "aarch64 GPU node", + "kind": { + "value": "host", + "provenance": "inferred", + "evidence": [ + "synthetic:engine-side kubectl get node: no hypervisor in node info" + ] + }, + "technology": { + "value": null, + "provenance": "not-applicable", + "reason": "the outermost host runs on hardware" + }, + "os": { + "family": { + "value": "linux", + "provenance": "observed", + "evidence": [ + "synthetic:platform.system" + ] + }, + "name": { + "value": "ubuntu", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side kubectl get node: status.nodeInfo.osImage" + ] + }, + "version": { + "value": "24.04", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side kubectl get node: status.nodeInfo.osImage" + ] + }, + "kernel": { + "value": "6.11.0-1016-nvidia", + "provenance": "observed", + "evidence": [ + "synthetic:platform.release" + ] + } + }, + "cpu": { + "arch": { + "value": "aarch64", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/sys/kernel/arch" + ] + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:engine-side kubectl get node: status.nodeInfo.architecture" + ] + } + }, + "service_manager": { + "name": { + "value": "systemd", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side node /run/systemd/system" + ] + }, + "scope": { + "value": "system", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side node /proc/1/comm" + ] + } + } + }, + { + "label": "Kubernetes pod", + "kind": { + "value": "container", + "provenance": "inferred", + "evidence": [ + "synthetic:/proc/self/cgroup kubepods", + "synthetic:KUBERNETES_SERVICE_HOST present" + ] + }, + "technology": { + "value": "containerd", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side kubectl get node: status.nodeInfo.containerRuntimeVersion" + ] + }, + "os": { + "family": { + "value": "linux", + "provenance": "observed", + "evidence": [ + "synthetic:platform.system" + ] + }, + "name": { + "value": "ubuntu", + "provenance": "observed", + "evidence": [ + "synthetic:/etc/os-release ID" + ] + }, + "version": { + "value": "24.04", + "provenance": "observed", + "evidence": [ + "synthetic:/etc/os-release VERSION_ID" + ] + }, + "kernel": { + "value": null, + "provenance": "not-applicable", + "reason": "a container shares the kernel of the layer below" + } + }, + "cpu": { + "arch": { + "value": "aarch64", + "provenance": "observed", + "evidence": [ + "synthetic:platform.machine" + ] + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:platform.machine", + "synthetic:/proc/sys/kernel/arch" + ] + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "systemd is not running as the service manager here; other managers are not probed" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "no service manager seen" + } + }, + "container": { + "engine_mode": { + "value": "rootful", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side containerd runs as root on the node" + ] + }, + "privileged": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:engine-side kubectl get pod: spec.containers[0].securityContext.privileged" + ] + }, + "devices": { + "value": [ + "nvidia-gpu" + ], + "provenance": "observed", + "evidence": [ + "synthetic:engine-side kubectl get pod: spec.containers[0].resources.limits" + ] + }, + "orchestration": { + "name": { + "value": "kubernetes", + "provenance": "inferred", + "evidence": [ + "synthetic:KUBERNETES_SERVICE_HOST present" + ] + }, + "version": { + "value": "1.33", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side kubectl version: serverVersion" + ] + }, + "scope": { + "value": null, + "provenance": "not-applicable", + "reason": "Kubernetes has no user or system scope" + } + } + } + } + ] + }, + "workload": { + "identity": { + "superuser": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "uid": { + "value": 1000, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "real_uid": { + "value": 1000, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "saved_uid": { + "value": 1000, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "gid": { + "value": 1000, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Gid" + ] + }, + "user_namespace": { + "initial": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/uid_map" + ] + }, + "uid_map": { + "value": [ + [ + 0, + 0, + 4294967295 + ] + ], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/uid_map" + ] + } + } + }, + "isolation": { + "no_new_privs": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status NoNewPrivs" + ] + }, + "capabilities": { + "effective": { + "value": [], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status CapEff" + ] + }, + "permitted": { + "value": [], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status CapPrm" + ] + }, + "bounding": { + "value": [ + "cap_chown", + "cap_dac_override", + "cap_fowner", + "cap_fsetid", + "cap_kill", + "cap_net_bind_service", + "cap_setfcap", + "cap_setgid", + "cap_setpcap", + "cap_setuid", + "cap_sys_chroot" + ], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status CapBnd" + ] + } + }, + "seccomp": { + "value": "filter", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Seccomp" + ] + }, + "apparmor": { + "enabled": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:/sys/module/apparmor/parameters/enabled" + ] + }, + "profile": { + "value": "cri-containerd.apparmor.d (enforce)", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/attr/apparmor/current" + ] + } + }, + "selinux": { + "mode": { + "value": "disabled", + "provenance": "inferred", + "evidence": [ + "synthetic:/proc/filesystems" + ] + }, + "context": { + "value": null, + "provenance": "not-applicable", + "reason": "SELinux is not in this kernel" + } + }, + "cgroup": { + "version": { + "value": "v2", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/cgroup" + ] + }, + "delegated": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:cgroup.procs writable" + ] + } + }, + "pid_namespace": { + "initial": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/ns/pid#inode" + ] + } + } + }, + "runtimes": { + "python": { + "implementation": { + "value": "cpython", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation.name" + ] + }, + "version": { + "value": "3.12.15", + "provenance": "observed", + "evidence": [ + "synthetic:platform.python_version" + ] + }, + "cache_tag": { + "value": "cpython-312", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation.cache_tag" + ] + }, + "soabi": { + "value": "cpython-312-aarch64-linux-gnu", + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.SOABI" + ] + }, + "platform": { + "value": "linux-aarch64", + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.get_platform" + ] + }, + "arch": { + "value": "aarch64", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation._multiarch" + ] + }, + "pointer_bits": { + "value": 64, + "provenance": "observed", + "evidence": [ + "synthetic:struct.calcsize" + ] + }, + "free_threaded_build": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.Py_GIL_DISABLED" + ] + }, + "gil_enabled": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:sys._is_gil_enabled" + ] + }, + "libc": { + "value": "glibc 2.39", + "provenance": "observed", + "evidence": [ + "synthetic:platform.libc_ver" + ] + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:sys.implementation._multiarch", + "synthetic:/proc/sys/kernel/arch" + ] + } + }, + "node": { + "version": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "modules_abi": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "napi_version": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "platform": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "arch": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + } + } + } + }, + "compute": { + "backends": [ + { + "id": "cpu", + "kind": "cpu", + "api": "cpu", + "available": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:collector ran on it" + ] + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "the CPU model is not recorded" + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "a CPU has no backend version" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "CPU features are not recorded" + } + }, + { + "id": "cuda", + "kind": "gpu", + "api": "cuda", + "available": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:nvidia-smi" + ] + }, + "device": { + "value": "NVIDIA GB10", + "provenance": "observed", + "evidence": [ + "synthetic:nvidia-smi" + ] + }, + "version": { + "value": "13.0", + "provenance": "observed", + "evidence": [ + "synthetic:nvidia-smi" + ] + }, + "features": { + "value": [ + "bf16", + "fp16" + ], + "provenance": "observed", + "evidence": [ + "synthetic:deviceQuery" + ] + }, + "extensions": { + "x-driver": "synthetic example of vendor data the validator ignores" + } + } + ], + "accelerator_inventory": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:nvidia-smi -L" + ] + } + }, + "stages": { + "detect": { + "status": "passed", + "evidence": [ + "synthetic:collector" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + } + }, + "install": { + "status": "passed", + "evidence": [ + "synthetic:uv pip sync log" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + } + }, + "initialize": { + "status": "passed", + "evidence": [ + "synthetic:manage.py check log" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + } + }, + "execute": { + "status": "passed", + "evidence": [ + "synthetic:inference smoke test log" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + }, + "compute": { + "requested": "cuda", + "effective": "cuda" + } + }, + "correctness": { + "status": "passed", + "evidence": [ + "synthetic:inference output comparison" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + }, + "compute": { + "requested": "cuda", + "effective": "cuda" + } + }, + "security": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "performance": { + "status": "not-run", + "reason": "not run in this scenario" + } + }, + "overall": "incomplete", + "probe_errors": [], + "extensions": { + "x-note": "Synthetic fixture for the validator's tests. It describes no machine that was measured." + } +} diff --git a/tests/compatibility/fixtures/linux-x64-vm.profile.json b/tests/compatibility/fixtures/linux-x64-vm.profile.json new file mode 100644 index 0000000..8b9167d --- /dev/null +++ b/tests/compatibility/fixtures/linux-x64-vm.profile.json @@ -0,0 +1,245 @@ +{ + "schema_version": 1, + "kind": "profile", + "synthetic": true, + "id": "synthetic-linux-x64-vm", + "title": "Synthetic Linux x86-64 virtual machine, CPU only", + "description": "Mirrors the shape of a GitHub-hosted runner record; used to test the link between a result and its profile.", + "support": { + "status": "not-assessed", + "lane": "Linux x86-64 glibc", + "reference": "docs/DEPENDENCY-STRATEGY.md#platform-and-accelerator-lanes" + }, + "evidence_sources": [], + "required_stages": [ + "detect" + ], + "environment": { + "layers": [ + { + "kind": { + "value": "host", + "provenance": "declared" + }, + "technology": { + "value": null, + "provenance": "not-applicable", + "reason": "the outermost host runs on hardware" + }, + "os": { + "family": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "name": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "version": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "kernel": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + }, + "cpu": { + "arch": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + } + }, + { + "label": "Virtual machine", + "kind": { + "value": "vm", + "provenance": "declared" + }, + "technology": { + "value": "hyper-v", + "provenance": "declared" + }, + "os": { + "family": { + "value": "linux", + "provenance": "declared" + }, + "name": { + "value": "ubuntu", + "provenance": "declared" + }, + "version": { + "value": "24.04", + "provenance": "declared" + }, + "kernel": { + "value": "6.11.0-1015-azure", + "provenance": "declared" + } + }, + "cpu": { + "arch": { + "value": "x86_64", + "provenance": "declared" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "whether the layer below emulates this CPU is not visible from inside it" + } + }, + "service_manager": { + "name": { + "value": "systemd", + "provenance": "declared" + }, + "scope": { + "value": "system", + "provenance": "declared" + } + } + } + ] + }, + "workload": { + "runtimes": { + "python": { + "implementation": { + "value": "cpython", + "provenance": "declared" + }, + "version": { + "value": "3.12.15", + "provenance": "declared" + }, + "cache_tag": { + "value": "cpython-312", + "provenance": "declared" + }, + "soabi": { + "value": "cpython-312-x86_64-linux-gnu", + "provenance": "declared" + }, + "platform": { + "value": "linux-x86_64", + "provenance": "declared" + }, + "arch": { + "value": "x86_64", + "provenance": "declared" + }, + "pointer_bits": { + "value": 64, + "provenance": "declared" + }, + "free_threaded_build": { + "value": false, + "provenance": "declared" + }, + "gil_enabled": { + "value": true, + "provenance": "declared" + }, + "libc": { + "value": null, + "provenance": "unknown", + "reason": "each run records it" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + }, + "node": { + "version": { + "value": "24.21.0", + "provenance": "declared" + }, + "modules_abi": { + "value": "137", + "provenance": "declared" + }, + "napi_version": { + "value": "10", + "provenance": "declared" + }, + "platform": { + "value": "linux", + "provenance": "declared" + }, + "arch": { + "value": "x86_64", + "provenance": "declared" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + } + } + }, + "compute": { + "backends": [ + { + "id": "cpu", + "kind": "cpu", + "api": "cpu", + "available": { + "value": true, + "provenance": "declared" + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "the CPU model is not recorded" + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "a CPU has no backend version" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "CPU features are not recorded" + } + } + ], + "accelerator_inventory": { + "value": null, + "provenance": "unknown", + "reason": "this collector does not look for GPUs or NPUs" + } + }, + "policy": { + "rules": {}, + "exceptions": [] + }, + "extensions": { + "x-note": "Synthetic fixture for the validator's tests. It describes no machine that was measured." + } +} diff --git a/tests/compatibility/fixtures/linux-x64-vm.result.json b/tests/compatibility/fixtures/linux-x64-vm.result.json new file mode 100644 index 0000000..6990110 --- /dev/null +++ b/tests/compatibility/fixtures/linux-x64-vm.result.json @@ -0,0 +1,651 @@ +{ + "schema_version": 1, + "kind": "result", + "synthetic": true, + "profile_id": "synthetic-linux-x64-vm", + "generated_at": null, + "collector": { + "name": "collect-environment", + "version": 1 + }, + "run": { + "ci": { + "system": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "event": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "repository": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "ref": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_id": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_attempt": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_url": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "workflow_ref": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "job": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_environment": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_os": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_arch": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "image_os": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "image_version": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + } + }, + "revision": { + "tested_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "event_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "pr_head_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "pr_base_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + } + } + }, + "environment": { + "layers": [ + { + "kind": { + "value": "host", + "provenance": "inferred", + "evidence": [ + "synthetic:/proc/cpuinfo flags hypervisor" + ] + }, + "technology": { + "value": null, + "provenance": "not-applicable", + "reason": "the outermost host runs on hardware" + }, + "os": { + "family": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "name": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "version": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "kernel": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + }, + "cpu": { + "arch": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + } + }, + { + "label": "Virtual machine", + "kind": { + "value": "vm", + "provenance": "inferred", + "evidence": [ + "synthetic:/proc/cpuinfo flags hypervisor" + ] + }, + "technology": { + "value": "hyper-v", + "provenance": "inferred", + "evidence": [ + "synthetic:/sys/class/dmi/id/sys_vendor" + ] + }, + "os": { + "family": { + "value": "linux", + "provenance": "observed", + "evidence": [ + "synthetic:platform.system" + ] + }, + "name": { + "value": "ubuntu", + "provenance": "observed", + "evidence": [ + "synthetic:/etc/os-release ID" + ] + }, + "version": { + "value": "24.04", + "provenance": "observed", + "evidence": [ + "synthetic:/etc/os-release VERSION_ID" + ] + }, + "kernel": { + "value": "6.11.0-1015-azure", + "provenance": "observed", + "evidence": [ + "synthetic:platform.release" + ] + } + }, + "cpu": { + "arch": { + "value": "x86_64", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/sys/kernel/arch" + ] + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "whether the layer below emulates this CPU is not visible from inside it" + } + }, + "service_manager": { + "name": { + "value": "systemd", + "provenance": "observed", + "evidence": [ + "synthetic:/run/systemd/system" + ] + }, + "scope": { + "value": "system", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/1/comm" + ] + } + } + } + ] + }, + "workload": { + "identity": { + "superuser": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "uid": { + "value": 1001, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "real_uid": { + "value": 1001, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "saved_uid": { + "value": 1001, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "gid": { + "value": 118, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Gid" + ] + }, + "user_namespace": { + "initial": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/uid_map" + ] + }, + "uid_map": { + "value": [ + [ + 0, + 0, + 4294967295 + ] + ], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/uid_map" + ] + } + } + }, + "isolation": { + "no_new_privs": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status NoNewPrivs" + ] + }, + "capabilities": { + "effective": { + "value": [], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status CapEff" + ] + }, + "permitted": { + "value": [], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status CapPrm" + ] + }, + "bounding": { + "value": [ + "cap_chown", + "cap_dac_override", + "cap_dac_read_search", + "cap_fowner", + "cap_fsetid", + "cap_kill", + "cap_setgid", + "cap_setuid", + "cap_setpcap", + "cap_linux_immutable", + "cap_net_bind_service", + "cap_net_broadcast", + "cap_net_admin", + "cap_net_raw", + "cap_ipc_lock", + "cap_ipc_owner", + "cap_sys_module", + "cap_sys_rawio", + "cap_sys_chroot", + "cap_sys_ptrace", + "cap_sys_pacct", + "cap_sys_admin", + "cap_sys_boot", + "cap_sys_nice", + "cap_sys_resource", + "cap_sys_time", + "cap_sys_tty_config", + "cap_mknod", + "cap_lease", + "cap_audit_write", + "cap_audit_control", + "cap_setfcap", + "cap_mac_override", + "cap_mac_admin", + "cap_syslog", + "cap_wake_alarm", + "cap_block_suspend", + "cap_audit_read", + "cap_perfmon", + "cap_bpf", + "cap_checkpoint_restore" + ], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status CapBnd" + ] + } + }, + "seccomp": { + "value": "disabled", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Seccomp" + ] + }, + "apparmor": { + "enabled": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:/sys/module/apparmor/parameters/enabled" + ] + }, + "profile": { + "value": "unconfined", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/attr/apparmor/current" + ] + } + }, + "selinux": { + "mode": { + "value": "disabled", + "provenance": "inferred", + "evidence": [ + "synthetic:/proc/filesystems" + ] + }, + "context": { + "value": null, + "provenance": "not-applicable", + "reason": "SELinux is not in this kernel" + } + }, + "cgroup": { + "version": { + "value": "v2", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/cgroup" + ] + }, + "delegated": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:cgroup.procs writable" + ] + } + }, + "pid_namespace": { + "initial": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/ns/pid#inode" + ] + } + } + }, + "runtimes": { + "python": { + "implementation": { + "value": "cpython", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation.name" + ] + }, + "version": { + "value": "3.12.15", + "provenance": "observed", + "evidence": [ + "synthetic:platform.python_version" + ] + }, + "cache_tag": { + "value": "cpython-312", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation.cache_tag" + ] + }, + "soabi": { + "value": "cpython-312-x86_64-linux-gnu", + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.SOABI" + ] + }, + "platform": { + "value": "linux-x86_64", + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.get_platform" + ] + }, + "arch": { + "value": "x86_64", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation._multiarch" + ] + }, + "pointer_bits": { + "value": 64, + "provenance": "observed", + "evidence": [ + "synthetic:struct.calcsize" + ] + }, + "free_threaded_build": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.Py_GIL_DISABLED" + ] + }, + "gil_enabled": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:sys._is_gil_enabled" + ] + }, + "libc": { + "value": "glibc 2.39", + "provenance": "observed", + "evidence": [ + "synthetic:platform.libc_ver" + ] + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:sys.implementation._multiarch", + "synthetic:/proc/sys/kernel/arch" + ] + } + }, + "node": { + "version": { + "value": "24.21.0", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "modules_abi": { + "value": "137", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "napi_version": { + "value": "10", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "platform": { + "value": "linux", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "arch": { + "value": "x86_64", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:node -p", + "synthetic:/proc/sys/kernel/arch" + ] + } + } + } + }, + "compute": { + "backends": [ + { + "id": "cpu", + "kind": "cpu", + "api": "cpu", + "available": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:collector ran on it" + ] + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "the CPU model is not recorded" + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "a CPU has no backend version" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "CPU features are not recorded" + } + } + ], + "accelerator_inventory": { + "value": null, + "provenance": "unknown", + "reason": "this collector does not look for GPUs or NPUs" + } + }, + "stages": { + "detect": { + "status": "passed", + "evidence": [ + "synthetic:collector" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + } + }, + "install": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "initialize": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "execute": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "correctness": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "security": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "performance": { + "status": "not-run", + "reason": "not run in this scenario" + } + }, + "overall": "incomplete", + "probe_errors": [], + "extensions": { + "x-note": "Synthetic fixture for the validator's tests. It describes no machine that was measured." + } +} diff --git a/tests/compatibility/fixtures/vm-podman-quadlet-systemd.result.json b/tests/compatibility/fixtures/vm-podman-quadlet-systemd.result.json new file mode 100644 index 0000000..703458f --- /dev/null +++ b/tests/compatibility/fixtures/vm-podman-quadlet-systemd.result.json @@ -0,0 +1,738 @@ +{ + "schema_version": 1, + "kind": "result", + "synthetic": true, + "profile_id": null, + "generated_at": null, + "collector": { + "name": "collect-environment", + "version": 1 + }, + "run": { + "ci": { + "system": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "event": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "repository": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "ref": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_id": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_attempt": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_url": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "workflow_ref": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "job": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_environment": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_os": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_arch": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "image_os": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "image_version": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + } + }, + "revision": { + "tested_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "event_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "pr_head_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "pr_base_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + } + } + }, + "environment": { + "layers": [ + { + "kind": { + "value": "host", + "provenance": "inferred", + "evidence": [ + "synthetic:/proc/cpuinfo flags hypervisor" + ] + }, + "technology": { + "value": null, + "provenance": "not-applicable", + "reason": "the outermost host runs on hardware" + }, + "os": { + "family": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "name": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "version": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "kernel": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + }, + "cpu": { + "arch": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + } + }, + { + "label": "KVM virtual machine", + "kind": { + "value": "vm", + "provenance": "inferred", + "evidence": [ + "synthetic:/proc/cpuinfo flags hypervisor", + "synthetic:/sys/class/dmi/id/sys_vendor" + ] + }, + "technology": { + "value": "kvm", + "provenance": "inferred", + "evidence": [ + "synthetic:/sys/class/dmi/id/sys_vendor" + ] + }, + "os": { + "family": { + "value": "linux", + "provenance": "observed", + "evidence": [ + "synthetic:platform.system" + ] + }, + "name": { + "value": "fedora", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side VM /etc/os-release ID" + ] + }, + "version": { + "value": "42", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side VM /etc/os-release VERSION_ID" + ] + }, + "kernel": { + "value": "6.14.4-300.fc42.x86_64", + "provenance": "observed", + "evidence": [ + "synthetic:platform.release" + ] + } + }, + "cpu": { + "arch": { + "value": "x86_64", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/sys/kernel/arch" + ] + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "whether the layer below emulates this CPU is not visible from inside it" + } + }, + "service_manager": { + "name": { + "value": "systemd", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side VM /run/systemd/system" + ] + }, + "scope": { + "value": "system", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side VM /proc/1/comm" + ] + } + } + }, + { + "label": "Podman container with systemd as PID 1", + "kind": { + "value": "container", + "provenance": "inferred", + "evidence": [ + "synthetic:/run/.containerenv" + ] + }, + "technology": { + "value": "podman", + "provenance": "inferred", + "evidence": [ + "synthetic:/run/.containerenv" + ] + }, + "os": { + "family": { + "value": "linux", + "provenance": "observed", + "evidence": [ + "synthetic:platform.system" + ] + }, + "name": { + "value": "fedora", + "provenance": "observed", + "evidence": [ + "synthetic:/etc/os-release ID" + ] + }, + "version": { + "value": "42", + "provenance": "observed", + "evidence": [ + "synthetic:/etc/os-release VERSION_ID" + ] + }, + "kernel": { + "value": null, + "provenance": "not-applicable", + "reason": "a container shares the kernel of the layer below" + } + }, + "cpu": { + "arch": { + "value": "x86_64", + "provenance": "observed", + "evidence": [ + "synthetic:platform.machine" + ] + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:platform.machine", + "synthetic:/proc/sys/kernel/arch" + ] + } + }, + "service_manager": { + "name": { + "value": "systemd", + "provenance": "observed", + "evidence": [ + "synthetic:/run/systemd/system" + ] + }, + "scope": { + "value": "system", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/1/comm" + ] + } + }, + "container": { + "engine_mode": { + "value": "rootless", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side podman info: host.security.rootless" + ] + }, + "privileged": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:engine-side podman inspect: HostConfig.Privileged" + ] + }, + "devices": { + "value": [], + "provenance": "observed", + "evidence": [ + "synthetic:engine-side podman inspect: HostConfig.Devices" + ] + }, + "orchestration": { + "name": { + "value": "quadlet", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side systemctl --user show: SourcePath is a .container unit" + ] + }, + "version": { + "value": "5.4", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side podman version" + ] + }, + "scope": { + "value": "user", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side systemctl --user show" + ] + } + } + } + } + ] + }, + "workload": { + "identity": { + "superuser": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "uid": { + "value": 1000, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "real_uid": { + "value": 1000, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "saved_uid": { + "value": 1000, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "gid": { + "value": 1000, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Gid" + ] + }, + "user_namespace": { + "initial": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/uid_map" + ] + }, + "uid_map": { + "value": [ + [ + 0, + 1000, + 1 + ], + [ + 1, + 524288, + 65536 + ] + ], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/uid_map" + ] + } + } + }, + "isolation": { + "no_new_privs": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status NoNewPrivs" + ] + }, + "capabilities": { + "effective": { + "value": [], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status CapEff" + ] + }, + "permitted": { + "value": [], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status CapPrm" + ] + }, + "bounding": { + "value": [ + "cap_chown", + "cap_dac_override", + "cap_fowner", + "cap_fsetid", + "cap_kill", + "cap_net_bind_service", + "cap_setfcap", + "cap_setgid", + "cap_setpcap", + "cap_setuid", + "cap_sys_chroot" + ], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status CapBnd" + ] + } + }, + "seccomp": { + "value": "filter", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Seccomp" + ] + }, + "apparmor": { + "enabled": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/sys/module/apparmor/parameters/enabled" + ] + }, + "profile": { + "value": null, + "provenance": "not-applicable", + "reason": "AppArmor is not enabled" + } + }, + "selinux": { + "mode": { + "value": "enforcing", + "provenance": "observed", + "evidence": [ + "synthetic:/sys/fs/selinux/enforce" + ] + }, + "context": { + "value": "system_u:system_r:container_t:s0:c12,c34", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/attr/selinux/current" + ] + } + }, + "cgroup": { + "version": { + "value": "v2", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/cgroup" + ] + }, + "delegated": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:cgroup.procs writable" + ] + } + }, + "pid_namespace": { + "initial": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/ns/pid#inode" + ] + } + } + }, + "runtimes": { + "python": { + "implementation": { + "value": "cpython", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation.name" + ] + }, + "version": { + "value": "3.12.15", + "provenance": "observed", + "evidence": [ + "synthetic:platform.python_version" + ] + }, + "cache_tag": { + "value": "cpython-312", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation.cache_tag" + ] + }, + "soabi": { + "value": "cpython-312-x86_64-linux-gnu", + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.SOABI" + ] + }, + "platform": { + "value": "linux-x86_64", + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.get_platform" + ] + }, + "arch": { + "value": "x86_64", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation._multiarch" + ] + }, + "pointer_bits": { + "value": 64, + "provenance": "observed", + "evidence": [ + "synthetic:struct.calcsize" + ] + }, + "free_threaded_build": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.Py_GIL_DISABLED" + ] + }, + "gil_enabled": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:sys._is_gil_enabled" + ] + }, + "libc": { + "value": "glibc 2.41", + "provenance": "observed", + "evidence": [ + "synthetic:platform.libc_ver" + ] + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:sys.implementation._multiarch", + "synthetic:/proc/sys/kernel/arch" + ] + } + }, + "node": { + "version": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "modules_abi": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "napi_version": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "platform": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "arch": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "node is not on PATH" + } + } + } + }, + "compute": { + "backends": [ + { + "id": "cpu", + "kind": "cpu", + "api": "cpu", + "available": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:collector ran on it" + ] + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "the CPU model is not recorded" + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "a CPU has no backend version" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "CPU features are not recorded" + } + } + ], + "accelerator_inventory": { + "value": null, + "provenance": "not-applicable", + "reason": "this lane uses the CPU only" + } + }, + "stages": { + "detect": { + "status": "passed", + "evidence": [ + "synthetic:collector" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + } + }, + "install": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "initialize": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "execute": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "correctness": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "security": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "performance": { + "status": "not-run", + "reason": "not run in this scenario" + } + }, + "overall": "incomplete", + "probe_errors": [], + "extensions": { + "x-note": "Synthetic fixture for the validator's tests. It describes no machine that was measured." + } +} diff --git a/tests/compatibility/fixtures/windows-arm64-native.profile.json b/tests/compatibility/fixtures/windows-arm64-native.profile.json new file mode 100644 index 0000000..9a14a18 --- /dev/null +++ b/tests/compatibility/fixtures/windows-arm64-native.profile.json @@ -0,0 +1,213 @@ +{ + "schema_version": 1, + "kind": "profile", + "synthetic": true, + "id": "synthetic-windows-arm64-native", + "title": "Synthetic Windows on Arm, native", + "description": "Native win_arm64 Python and Node with the CPU and a Qualcomm NPU through QNN. docs/DEPENDENCY-STRATEGY.md records the lane as not supported.", + "support": { + "status": "not-supported", + "lane": "Windows on Arm, native (`win_arm64`)", + "reference": "docs/DEPENDENCY-STRATEGY.md#platform-and-accelerator-lanes" + }, + "evidence_sources": [], + "required_stages": [ + "detect" + ], + "environment": { + "layers": [ + { + "label": "Windows on Arm PC", + "kind": { + "value": "host", + "provenance": "declared" + }, + "technology": { + "value": null, + "provenance": "not-applicable", + "reason": "the outermost host runs on hardware" + }, + "os": { + "family": { + "value": "windows", + "provenance": "declared" + }, + "name": { + "value": "windows", + "provenance": "declared" + }, + "version": { + "value": "10.0.26100", + "provenance": "declared" + }, + "kernel": { + "value": "10.0.26100", + "provenance": "declared" + } + }, + "cpu": { + "arch": { + "value": "aarch64", + "provenance": "declared" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + }, + "service_manager": { + "name": { + "value": "windows-scm", + "provenance": "declared" + }, + "scope": { + "value": "system", + "provenance": "declared" + } + } + } + ] + }, + "workload": { + "runtimes": { + "python": { + "implementation": { + "value": "cpython", + "provenance": "declared" + }, + "version": { + "value": "3.12.15", + "provenance": "declared" + }, + "cache_tag": { + "value": "cpython-312", + "provenance": "declared" + }, + "soabi": { + "value": "cp312-win_arm64", + "provenance": "declared" + }, + "platform": { + "value": "win-arm64", + "provenance": "declared" + }, + "arch": { + "value": "aarch64", + "provenance": "declared" + }, + "pointer_bits": { + "value": 64, + "provenance": "declared" + }, + "free_threaded_build": { + "value": false, + "provenance": "declared" + }, + "gil_enabled": { + "value": true, + "provenance": "declared" + }, + "libc": { + "value": null, + "provenance": "not-applicable", + "reason": "no C library version on this OS" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + }, + "node": { + "version": { + "value": "24.21.0", + "provenance": "declared" + }, + "modules_abi": { + "value": "137", + "provenance": "declared" + }, + "napi_version": { + "value": "10", + "provenance": "declared" + }, + "platform": { + "value": "win32", + "provenance": "declared" + }, + "arch": { + "value": "aarch64", + "provenance": "declared" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + } + } + }, + "compute": { + "backends": [ + { + "id": "cpu", + "kind": "cpu", + "api": "cpu", + "available": { + "value": true, + "provenance": "declared" + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "the CPU model is not recorded" + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "a CPU has no backend version" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "CPU features are not recorded" + } + }, + { + "id": "qnn-npu", + "kind": "npu", + "api": "qnn", + "available": { + "value": null, + "provenance": "unknown", + "reason": "not probed" + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "not probed" + }, + "version": { + "value": null, + "provenance": "unknown", + "reason": "not probed" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "not probed" + } + } + ], + "accelerator_inventory": { + "value": null, + "provenance": "unknown", + "reason": "only the NPU named in the profile was listed" + } + }, + "policy": { + "rules": {}, + "exceptions": [] + }, + "extensions": { + "x-note": "Synthetic fixture for the validator's tests. It describes no machine that was measured." + } +} diff --git a/tests/compatibility/fixtures/windows-arm64-native.result.json b/tests/compatibility/fixtures/windows-arm64-native.result.json new file mode 100644 index 0000000..966fbee --- /dev/null +++ b/tests/compatibility/fixtures/windows-arm64-native.result.json @@ -0,0 +1,541 @@ +{ + "schema_version": 1, + "kind": "result", + "synthetic": true, + "profile_id": "synthetic-windows-arm64-native", + "generated_at": null, + "collector": { + "name": "collect-environment", + "version": 1 + }, + "run": { + "ci": { + "system": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "event": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "repository": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "ref": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_id": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_attempt": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_url": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "workflow_ref": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "job": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_environment": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_os": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_arch": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "image_os": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "image_version": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + } + }, + "revision": { + "tested_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "event_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "pr_head_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "pr_base_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + } + } + }, + "environment": { + "layers": [ + { + "label": "Windows on Arm PC", + "kind": { + "value": "host", + "provenance": "inferred", + "evidence": [ + "synthetic:Win32_ComputerSystem HypervisorPresent false" + ] + }, + "technology": { + "value": null, + "provenance": "not-applicable", + "reason": "the outermost host runs on hardware" + }, + "os": { + "family": { + "value": "windows", + "provenance": "observed", + "evidence": [ + "synthetic:platform.system" + ] + }, + "name": { + "value": "windows", + "provenance": "observed", + "evidence": [ + "synthetic:platform.system" + ] + }, + "version": { + "value": "10.0.26100", + "provenance": "observed", + "evidence": [ + "synthetic:platform.version" + ] + }, + "kernel": { + "value": "10.0.26100", + "provenance": "observed", + "evidence": [ + "synthetic:platform.version" + ] + } + }, + "cpu": { + "arch": { + "value": "aarch64", + "provenance": "observed", + "evidence": [ + "synthetic:platform.machine" + ] + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:IsWow64Process2 native machine" + ] + } + }, + "service_manager": { + "name": { + "value": "windows-scm", + "provenance": "observed", + "evidence": [ + "synthetic:service control manager" + ] + }, + "scope": { + "value": "system", + "provenance": "observed", + "evidence": [ + "synthetic:service control manager" + ] + } + } + } + ] + }, + "workload": { + "identity": { + "superuser": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:token elevation" + ] + }, + "uid": { + "value": null, + "provenance": "not-applicable", + "reason": "Windows has no POSIX user ID or user namespace" + }, + "real_uid": { + "value": null, + "provenance": "not-applicable", + "reason": "Windows has no POSIX user IDs" + }, + "saved_uid": { + "value": null, + "provenance": "not-applicable", + "reason": "Windows has no POSIX user IDs" + }, + "gid": { + "value": null, + "provenance": "not-applicable", + "reason": "Windows has no POSIX user ID or user namespace" + }, + "user_namespace": { + "initial": { + "value": null, + "provenance": "not-applicable", + "reason": "Windows has no POSIX user ID or user namespace" + }, + "uid_map": { + "value": null, + "provenance": "not-applicable", + "reason": "Windows has no POSIX user ID or user namespace" + } + } + }, + "isolation": { + "no_new_privs": { + "value": null, + "provenance": "not-applicable", + "reason": "a Linux interface; not applicable on Windows" + }, + "capabilities": { + "effective": { + "value": null, + "provenance": "not-applicable", + "reason": "a Linux interface; not applicable on Windows" + }, + "permitted": { + "value": null, + "provenance": "not-applicable", + "reason": "a Linux interface; not applicable on Windows" + }, + "bounding": { + "value": null, + "provenance": "not-applicable", + "reason": "a Linux interface; not applicable on Windows" + } + }, + "seccomp": { + "value": null, + "provenance": "not-applicable", + "reason": "a Linux interface; not applicable on Windows" + }, + "apparmor": { + "enabled": { + "value": null, + "provenance": "not-applicable", + "reason": "a Linux interface; not applicable on Windows" + }, + "profile": { + "value": null, + "provenance": "not-applicable", + "reason": "a Linux interface; not applicable on Windows" + } + }, + "selinux": { + "mode": { + "value": null, + "provenance": "not-applicable", + "reason": "a Linux interface; not applicable on Windows" + }, + "context": { + "value": null, + "provenance": "not-applicable", + "reason": "a Linux interface; not applicable on Windows" + } + }, + "cgroup": { + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "a Linux interface; not applicable on Windows" + }, + "delegated": { + "value": null, + "provenance": "not-applicable", + "reason": "a Linux interface; not applicable on Windows" + } + }, + "pid_namespace": { + "initial": { + "value": null, + "provenance": "not-applicable", + "reason": "a Linux interface; not applicable on Windows" + } + } + }, + "runtimes": { + "python": { + "implementation": { + "value": "cpython", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation.name" + ] + }, + "version": { + "value": "3.12.15", + "provenance": "observed", + "evidence": [ + "synthetic:platform.python_version" + ] + }, + "cache_tag": { + "value": "cpython-312", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation.cache_tag" + ] + }, + "soabi": { + "value": "cp312-win_arm64", + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.SOABI" + ] + }, + "platform": { + "value": "win-arm64", + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.get_platform" + ] + }, + "arch": { + "value": "aarch64", + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.get_platform" + ] + }, + "pointer_bits": { + "value": 64, + "provenance": "observed", + "evidence": [ + "synthetic:struct.calcsize" + ] + }, + "free_threaded_build": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.Py_GIL_DISABLED" + ] + }, + "gil_enabled": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:sys._is_gil_enabled" + ] + }, + "libc": { + "value": null, + "provenance": "not-applicable", + "reason": "no C library version on this OS" + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:platform.machine", + "synthetic:sysconfig.get_platform" + ] + } + }, + "node": { + "version": { + "value": "24.21.0", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "modules_abi": { + "value": "137", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "napi_version": { + "value": "10", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "platform": { + "value": "win32", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "arch": { + "value": "aarch64", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:platform.machine", + "synthetic:node -p" + ] + } + } + } + }, + "compute": { + "backends": [ + { + "id": "cpu", + "kind": "cpu", + "api": "cpu", + "available": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:collector ran on it" + ] + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "the CPU model is not recorded" + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "a CPU has no backend version" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "CPU features are not recorded" + } + }, + { + "id": "qnn-npu", + "kind": "npu", + "api": "qnn", + "available": { + "value": null, + "provenance": "unknown", + "reason": "not probed" + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "not probed" + }, + "version": { + "value": null, + "provenance": "unknown", + "reason": "not probed" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "not probed" + } + } + ], + "accelerator_inventory": { + "value": null, + "provenance": "unknown", + "reason": "only the NPU named in the profile was listed" + } + }, + "stages": { + "detect": { + "status": "passed", + "evidence": [ + "synthetic:collector" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + } + }, + "install": { + "status": "failed", + "reason": "the backend lock has no win_arm64 wheels for autobahn, cryptography and psycopg-binary", + "evidence": [ + "synthetic:uv pip sync log" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + } + }, + "initialize": { + "status": "blocked", + "reason": "install failed" + }, + "execute": { + "status": "blocked", + "reason": "install failed" + }, + "correctness": { + "status": "blocked", + "reason": "install failed" + }, + "security": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "performance": { + "status": "blocked", + "reason": "install failed" + } + }, + "overall": "failed", + "probe_errors": [], + "extensions": { + "x-note": "Synthetic fixture for the validator's tests. It describes no machine that was measured." + } +} diff --git a/tests/compatibility/fixtures/windows-arm64-wsl2-podman.profile.json b/tests/compatibility/fixtures/windows-arm64-wsl2-podman.profile.json new file mode 100644 index 0000000..27857fe --- /dev/null +++ b/tests/compatibility/fixtures/windows-arm64-wsl2-podman.profile.json @@ -0,0 +1,337 @@ +{ + "schema_version": 1, + "kind": "profile", + "synthetic": true, + "id": "synthetic-windows-arm64-wsl2-podman", + "title": "Synthetic Windows on Arm, WSL2 Ubuntu arm64, rootless Podman", + "description": "A non-root user in a rootless Podman container in WSL2 on Windows on Arm; the engine is required to be rootless.", + "support": { + "status": "statically-resolvable", + "lane": "Windows on Arm with WSL2 (Ubuntu arm64)", + "reference": "docs/DEPENDENCY-STRATEGY.md#platform-and-accelerator-lanes" + }, + "evidence_sources": [], + "required_stages": [ + "detect", + "install", + "initialize", + "execute", + "correctness" + ], + "environment": { + "layers": [ + { + "label": "Windows on Arm PC", + "kind": { + "value": "host", + "provenance": "declared" + }, + "technology": { + "value": null, + "provenance": "not-applicable", + "reason": "the outermost host runs on hardware" + }, + "os": { + "family": { + "value": "windows", + "provenance": "declared" + }, + "name": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "version": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "kernel": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + }, + "cpu": { + "arch": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + } + }, + { + "label": "WSL2 utility VM", + "kind": { + "value": "wsl2", + "provenance": "declared" + }, + "technology": { + "value": "wsl2", + "provenance": "declared" + }, + "os": { + "family": { + "value": "linux", + "provenance": "declared" + }, + "name": { + "value": null, + "provenance": "unknown", + "reason": "not visible from inside the container" + }, + "version": { + "value": null, + "provenance": "unknown", + "reason": "not visible from inside the container" + }, + "kernel": { + "value": "6.6.87.2-microsoft-standard-WSL2", + "provenance": "declared" + } + }, + "cpu": { + "arch": { + "value": "aarch64", + "provenance": "declared" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "whether the layer below emulates this CPU is not visible from inside it" + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "not visible from inside the container" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "not visible from inside the container" + } + } + }, + { + "label": "Podman container", + "kind": { + "value": "container", + "provenance": "declared" + }, + "technology": { + "value": "podman", + "provenance": "declared" + }, + "os": { + "family": { + "value": "linux", + "provenance": "declared" + }, + "name": { + "value": "ubuntu", + "provenance": "declared" + }, + "version": { + "value": "24.04", + "provenance": "declared" + }, + "kernel": { + "value": null, + "provenance": "not-applicable", + "reason": "a container shares the kernel of the layer below" + } + }, + "cpu": { + "arch": { + "value": "aarch64", + "provenance": "declared" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "systemd is not running as the service manager here; other managers are not probed" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "no service manager seen" + } + }, + "container": { + "engine_mode": { + "value": "rootless", + "provenance": "declared" + }, + "privileged": { + "value": false, + "provenance": "declared" + }, + "devices": { + "value": [], + "provenance": "declared" + }, + "orchestration": { + "name": { + "value": "none", + "provenance": "declared" + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "no orchestrator" + }, + "scope": { + "value": null, + "provenance": "not-applicable", + "reason": "no orchestrator" + } + } + } + } + ] + }, + "workload": { + "runtimes": { + "python": { + "implementation": { + "value": "cpython", + "provenance": "declared" + }, + "version": { + "value": "3.12.15", + "provenance": "declared" + }, + "cache_tag": { + "value": "cpython-312", + "provenance": "declared" + }, + "soabi": { + "value": "cpython-312-aarch64-linux-gnu", + "provenance": "declared" + }, + "platform": { + "value": "linux-aarch64", + "provenance": "declared" + }, + "arch": { + "value": "aarch64", + "provenance": "declared" + }, + "pointer_bits": { + "value": 64, + "provenance": "declared" + }, + "free_threaded_build": { + "value": false, + "provenance": "declared" + }, + "gil_enabled": { + "value": true, + "provenance": "declared" + }, + "libc": { + "value": null, + "provenance": "unknown", + "reason": "each run records it" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + }, + "node": { + "version": { + "value": "24.21.0", + "provenance": "declared" + }, + "modules_abi": { + "value": "137", + "provenance": "declared" + }, + "napi_version": { + "value": "10", + "provenance": "declared" + }, + "platform": { + "value": "linux", + "provenance": "declared" + }, + "arch": { + "value": "aarch64", + "provenance": "declared" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + } + } + }, + "compute": { + "backends": [ + { + "id": "cpu", + "kind": "cpu", + "api": "cpu", + "available": { + "value": true, + "provenance": "declared" + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "the CPU model is not recorded" + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "a CPU has no backend version" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "CPU features are not recorded" + } + } + ], + "accelerator_inventory": { + "value": null, + "provenance": "not-applicable", + "reason": "this lane uses the CPU only" + } + }, + "policy": { + "rules": { + "engine-rootless": "required" + }, + "exceptions": [] + }, + "extensions": { + "x-note": "Synthetic fixture for the validator's tests. It describes no machine that was measured." + } +} diff --git a/tests/compatibility/fixtures/windows-arm64-wsl2-podman.result.json b/tests/compatibility/fixtures/windows-arm64-wsl2-podman.result.json new file mode 100644 index 0000000..5ad4b11 --- /dev/null +++ b/tests/compatibility/fixtures/windows-arm64-wsl2-podman.result.json @@ -0,0 +1,767 @@ +{ + "schema_version": 1, + "kind": "result", + "synthetic": true, + "profile_id": "synthetic-windows-arm64-wsl2-podman", + "generated_at": null, + "collector": { + "name": "collect-environment", + "version": 1 + }, + "run": { + "ci": { + "system": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "event": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "repository": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "ref": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_id": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_attempt": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "run_url": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "workflow_ref": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "job": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_environment": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_os": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "runner_arch": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "image_os": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "image_version": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + } + }, + "revision": { + "tested_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "event_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "pr_head_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + }, + "pr_base_sha": { + "value": null, + "provenance": "not-applicable", + "reason": "synthetic fixture; there was no run" + } + } + }, + "environment": { + "layers": [ + { + "label": "Windows on Arm PC", + "kind": { + "value": "host", + "provenance": "inferred", + "evidence": [ + "synthetic:/proc/sys/kernel/osrelease" + ] + }, + "technology": { + "value": null, + "provenance": "not-applicable", + "reason": "the outermost host runs on hardware" + }, + "os": { + "family": { + "value": "windows", + "provenance": "inferred", + "evidence": [ + "synthetic:/proc/sys/kernel/osrelease" + ] + }, + "name": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "version": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "kernel": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + }, + "cpu": { + "arch": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "a guest cannot see the physical host" + } + } + }, + { + "label": "WSL2 utility VM", + "kind": { + "value": "wsl2", + "provenance": "inferred", + "evidence": [ + "synthetic:/proc/sys/kernel/osrelease" + ] + }, + "technology": { + "value": "wsl2", + "provenance": "inferred", + "evidence": [ + "synthetic:/proc/sys/kernel/osrelease" + ] + }, + "os": { + "family": { + "value": "linux", + "provenance": "observed", + "evidence": [ + "synthetic:platform.system" + ] + }, + "name": { + "value": null, + "provenance": "unknown", + "reason": "not visible from inside the container" + }, + "version": { + "value": null, + "provenance": "unknown", + "reason": "not visible from inside the container" + }, + "kernel": { + "value": "6.6.87.2-microsoft-standard-WSL2", + "provenance": "observed", + "evidence": [ + "synthetic:platform.release" + ] + } + }, + "cpu": { + "arch": { + "value": "aarch64", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/sys/kernel/arch" + ] + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "whether the layer below emulates this CPU is not visible from inside it" + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "not visible from inside the container" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "not visible from inside the container" + } + } + }, + { + "label": "Podman container", + "kind": { + "value": "container", + "provenance": "inferred", + "evidence": [ + "synthetic:/run/.containerenv" + ] + }, + "technology": { + "value": "podman", + "provenance": "inferred", + "evidence": [ + "synthetic:/run/.containerenv" + ] + }, + "os": { + "family": { + "value": "linux", + "provenance": "observed", + "evidence": [ + "synthetic:platform.system" + ] + }, + "name": { + "value": "ubuntu", + "provenance": "observed", + "evidence": [ + "synthetic:/etc/os-release ID" + ] + }, + "version": { + "value": "24.04", + "provenance": "observed", + "evidence": [ + "synthetic:/etc/os-release VERSION_ID" + ] + }, + "kernel": { + "value": null, + "provenance": "not-applicable", + "reason": "a container shares the kernel of the layer below" + } + }, + "cpu": { + "arch": { + "value": "aarch64", + "provenance": "observed", + "evidence": [ + "synthetic:platform.machine" + ] + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:platform.machine", + "synthetic:/proc/sys/kernel/arch" + ] + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "systemd is not running as the service manager here; other managers are not probed" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "no service manager seen" + } + }, + "container": { + "engine_mode": { + "value": "rootless", + "provenance": "observed", + "evidence": [ + "synthetic:engine-side podman info: host.security.rootless" + ] + }, + "privileged": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:engine-side podman inspect: HostConfig.Privileged" + ] + }, + "devices": { + "value": [], + "provenance": "observed", + "evidence": [ + "synthetic:engine-side podman inspect: HostConfig.Devices" + ] + }, + "orchestration": { + "name": { + "value": "none", + "provenance": "inferred", + "evidence": [ + "synthetic:engine-side podman inspect: no orchestrator labels" + ] + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "no orchestrator" + }, + "scope": { + "value": null, + "provenance": "not-applicable", + "reason": "no orchestrator" + } + } + } + } + ] + }, + "workload": { + "identity": { + "superuser": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "uid": { + "value": 1000, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "real_uid": { + "value": 1000, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "saved_uid": { + "value": 1000, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Uid" + ] + }, + "gid": { + "value": 1000, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Gid" + ] + }, + "user_namespace": { + "initial": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/uid_map" + ] + }, + "uid_map": { + "value": [ + [ + 0, + 1000, + 1 + ], + [ + 1, + 100000, + 65536 + ] + ], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/uid_map" + ] + } + } + }, + "isolation": { + "no_new_privs": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status NoNewPrivs" + ] + }, + "capabilities": { + "effective": { + "value": [], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status CapEff" + ] + }, + "permitted": { + "value": [], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status CapPrm" + ] + }, + "bounding": { + "value": [ + "cap_chown", + "cap_dac_override", + "cap_fowner", + "cap_fsetid", + "cap_kill", + "cap_net_bind_service", + "cap_setfcap", + "cap_setgid", + "cap_setpcap", + "cap_setuid", + "cap_sys_chroot" + ], + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status CapBnd" + ] + } + }, + "seccomp": { + "value": "filter", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/status Seccomp" + ] + }, + "apparmor": { + "enabled": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/sys/module/apparmor/parameters/enabled" + ] + }, + "profile": { + "value": null, + "provenance": "not-applicable", + "reason": "AppArmor is not enabled" + } + }, + "selinux": { + "mode": { + "value": "disabled", + "provenance": "inferred", + "evidence": [ + "synthetic:/proc/filesystems" + ] + }, + "context": { + "value": null, + "provenance": "not-applicable", + "reason": "SELinux is not in this kernel" + } + }, + "cgroup": { + "version": { + "value": "v2", + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/cgroup" + ] + }, + "delegated": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:cgroup.procs writable" + ] + } + }, + "pid_namespace": { + "initial": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:/proc/self/ns/pid#inode" + ] + } + } + }, + "runtimes": { + "python": { + "implementation": { + "value": "cpython", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation.name" + ] + }, + "version": { + "value": "3.12.15", + "provenance": "observed", + "evidence": [ + "synthetic:platform.python_version" + ] + }, + "cache_tag": { + "value": "cpython-312", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation.cache_tag" + ] + }, + "soabi": { + "value": "cpython-312-aarch64-linux-gnu", + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.SOABI" + ] + }, + "platform": { + "value": "linux-aarch64", + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.get_platform" + ] + }, + "arch": { + "value": "aarch64", + "provenance": "observed", + "evidence": [ + "synthetic:sys.implementation._multiarch" + ] + }, + "pointer_bits": { + "value": 64, + "provenance": "observed", + "evidence": [ + "synthetic:struct.calcsize" + ] + }, + "free_threaded_build": { + "value": false, + "provenance": "observed", + "evidence": [ + "synthetic:sysconfig.Py_GIL_DISABLED" + ] + }, + "gil_enabled": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:sys._is_gil_enabled" + ] + }, + "libc": { + "value": "glibc 2.39", + "provenance": "observed", + "evidence": [ + "synthetic:platform.libc_ver" + ] + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:sys.implementation._multiarch", + "synthetic:/proc/sys/kernel/arch" + ] + } + }, + "node": { + "version": { + "value": "24.21.0", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "modules_abi": { + "value": "137", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "napi_version": { + "value": "10", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "platform": { + "value": "linux", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "arch": { + "value": "aarch64", + "provenance": "observed", + "evidence": [ + "synthetic:node -p" + ] + }, + "execution": { + "value": "native", + "provenance": "inferred", + "evidence": [ + "synthetic:node -p", + "synthetic:/proc/sys/kernel/arch" + ] + } + } + } + }, + "compute": { + "backends": [ + { + "id": "cpu", + "kind": "cpu", + "api": "cpu", + "available": { + "value": true, + "provenance": "observed", + "evidence": [ + "synthetic:collector ran on it" + ] + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "the CPU model is not recorded" + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "a CPU has no backend version" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "CPU features are not recorded" + } + } + ], + "accelerator_inventory": { + "value": null, + "provenance": "not-applicable", + "reason": "this lane uses the CPU only" + } + }, + "stages": { + "detect": { + "status": "passed", + "evidence": [ + "synthetic:collector" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + } + }, + "install": { + "status": "passed", + "evidence": [ + "synthetic:uv pip sync log" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + } + }, + "initialize": { + "status": "passed", + "evidence": [ + "synthetic:manage.py check log" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + } + }, + "execute": { + "status": "passed", + "evidence": [ + "synthetic:manage.py test log" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + } + }, + "correctness": { + "status": "passed", + "evidence": [ + "synthetic:manage.py test summary" + ], + "privilege": { + "escalation": { + "value": "none", + "provenance": "declared" + } + } + }, + "security": { + "status": "not-run", + "reason": "not run in this scenario" + }, + "performance": { + "status": "not-run", + "reason": "not run in this scenario" + } + }, + "overall": "incomplete", + "probe_errors": [], + "extensions": { + "x-note": "Synthetic fixture for the validator's tests. It describes no machine that was measured." + } +} diff --git a/tests/compatibility/profiles/linux-x64-gha.json b/tests/compatibility/profiles/linux-x64-gha.json new file mode 100644 index 0000000..c4017f1 --- /dev/null +++ b/tests/compatibility/profiles/linux-x64-gha.json @@ -0,0 +1,269 @@ +{ + "schema_version": 1, + "kind": "profile", + "synthetic": false, + "id": "linux-x64-gha", + "title": "Linux x86-64 on a GitHub-hosted ubuntu-24.04 runner", + "description": "The checked lane as .github/workflows/ci.yml runs it: CPython 3.12.15 from uv and Node 24.21.0 from .nvmrc on a GitHub-hosted ubuntu-24.04 x86-64 virtual machine, as a non-root user, CPU only.", + "support": { + "status": "checked", + "lane": "Linux x86-64 glibc", + "reference": "docs/DEPENDENCY-STRATEGY.md#platform-and-accelerator-lanes" + }, + "evidence_sources": [ + { + "workflow": ".github/workflows/ci.yml", + "job": "compatibility" + } + ], + "runner": { + "runner_environment": { + "value": "github-hosted", + "provenance": "declared" + }, + "runner_os": { + "value": "Linux", + "provenance": "declared" + }, + "runner_arch": { + "value": "X64", + "provenance": "declared" + }, + "image_os": { + "value": "ubuntu24", + "provenance": "declared" + } + }, + "required_stages": [ + "detect" + ], + "environment": { + "layers": [ + { + "label": "Physical host, not visible to the runner", + "kind": { + "value": "host", + "provenance": "declared" + }, + "technology": { + "value": null, + "provenance": "not-applicable", + "reason": "the outermost host runs on hardware" + }, + "os": { + "family": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + }, + "name": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + }, + "version": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + }, + "kernel": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + } + }, + "cpu": { + "arch": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + } + } + }, + { + "label": "GitHub-hosted runner virtual machine (runs-on: ubuntu-24.04)", + "kind": { + "value": "vm", + "provenance": "declared" + }, + "technology": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the hypervisor; each run records what DMI shows" + }, + "os": { + "family": { + "value": "linux", + "provenance": "declared" + }, + "name": { + "value": "ubuntu", + "provenance": "declared" + }, + "version": { + "value": "24.04", + "provenance": "declared" + }, + "kernel": { + "value": null, + "provenance": "unknown", + "reason": "changes with runner image updates; each run records it" + } + }, + "cpu": { + "arch": { + "value": "x86_64", + "provenance": "declared" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "whether the hypervisor emulates the CPU is not visible from inside the VM" + } + }, + "service_manager": { + "name": { + "value": "systemd", + "provenance": "declared" + }, + "scope": { + "value": "system", + "provenance": "declared" + } + } + } + ] + }, + "workload": { + "runtimes": { + "python": { + "implementation": { + "value": "cpython", + "provenance": "declared" + }, + "version": { + "value": "3.12.15", + "provenance": "declared" + }, + "cache_tag": { + "value": "cpython-312", + "provenance": "declared" + }, + "soabi": { + "value": "cpython-312-x86_64-linux-gnu", + "provenance": "declared" + }, + "platform": { + "value": "linux-x86_64", + "provenance": "declared" + }, + "arch": { + "value": "x86_64", + "provenance": "declared" + }, + "pointer_bits": { + "value": 64, + "provenance": "declared" + }, + "free_threaded_build": { + "value": false, + "provenance": "declared" + }, + "gil_enabled": { + "value": true, + "provenance": "declared" + }, + "libc": { + "value": "glibc 2.39", + "provenance": "declared" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + }, + "node": { + "version": { + "value": "24.21.0", + "provenance": "declared" + }, + "modules_abi": { + "value": null, + "provenance": "unknown", + "reason": "follows from the Node version; each run records it" + }, + "napi_version": { + "value": null, + "provenance": "unknown", + "reason": "follows from the Node version; each run records it" + }, + "platform": { + "value": "linux", + "provenance": "declared" + }, + "arch": { + "value": "x86_64", + "provenance": "declared" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + } + } + }, + "compute": { + "backends": [ + { + "id": "cpu", + "kind": "cpu", + "api": "cpu", + "available": { + "value": true, + "provenance": "declared" + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not fix the CPU model of hosted runners" + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "a CPU has no backend version" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "not declared" + } + } + ], + "accelerator_inventory": { + "value": null, + "provenance": "not-applicable", + "reason": "this lane uses the CPU only" + } + }, + "policy": { + "rules": {}, + "exceptions": [] + } +} diff --git a/tests/compatibility/schema.json b/tests/compatibility/schema.json new file mode 100644 index 0000000..6cd3183 --- /dev/null +++ b/tests/compatibility/schema.json @@ -0,0 +1,1617 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/P-R-N-D/Claus-AI/blob/main/tests/compatibility/schema.json", + "title": "Claus compatibility profile or result", + "$comment": "Uses only the keywords that scripts/ci/validate_compatibility.py implements; it rejects a schema with any other keyword. Rules across fields are in the validator, not here. See docs/CI-MATRIX.md.", + "type": "object", + "required": [ + "schema_version", + "kind" + ], + "properties": { + "kind": { + "enum": [ + "profile", + "result" + ] + } + }, + "if": { + "properties": { + "kind": { + "const": "profile" + } + } + }, + "then": { + "$ref": "#/$defs/profile" + }, + "else": { + "$ref": "#/$defs/result" + }, + "$defs": { + "text": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "slug": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._+-]{0,63}$" + }, + "provenance": { + "description": "observed: read on the machine by the collector, with evidence; inferred: derived from named evidence; declared: stated by a profile, workflow or tool, not measured; unknown and not-applicable: no value, with a reason.", + "enum": [ + "observed", + "inferred", + "declared", + "unknown", + "not-applicable" + ] + }, + "evidence_ref": { + "description": "Where a value came from, as :. synthetic: marks fixtures that were never measured.", + "type": "string", + "pattern": "^(file|env|python|cmd|event|git|tool|doc|profile|workflow|log|artifact|synthetic):[!-~][ -~]{0,299}$" + }, + "evidence_list": { + "type": "array", + "items": { + "$ref": "#/$defs/evidence_ref" + }, + "maxItems": 20 + }, + "extensions": { + "description": "Free-form data under x- names. The validator checks its strings for control characters and secret-shaped values and nothing else; it never treats an extension as evidence of support.", + "type": "object", + "propertyNames": { + "pattern": "^x-[a-z0-9][a-z0-9-]{0,62}$" + }, + "additionalProperties": {} + }, + "fact_string": { + "description": "A string fact.", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "type": [ + "string", + "null" + ], + "minLength": 1, + "maxLength": 200 + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_slug": { + "description": "A lower-case identifier fact; values outside the documented list are accepted with a warning.", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "type": [ + "string", + "null" + ], + "pattern": "^[a-z0-9][a-z0-9._+-]{0,63}$" + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_integer": { + "description": "A non-negative integer fact.", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 4294967295 + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_boolean": { + "description": "A boolean fact.", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "type": [ + "boolean", + "null" + ] + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_sha": { + "description": "A Git commit SHA-1 fact.", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "type": [ + "string", + "null" + ], + "pattern": "^[0-9a-f]{40}$" + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_slug_list": { + "description": "A list of identifiers.", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/$defs/slug" + }, + "maxItems": 64 + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_uid_map": { + "description": "Lines of /proc/self/uid_map as [inside, outside, count].", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "type": [ + "array", + "null" + ], + "maxItems": 340, + "items": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295 + } + } + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_layer_kind": { + "description": "What kind of execution layer this is.", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "enum": [ + "host", + "vm", + "wsl2", + "container", + null + ] + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_execution": { + "description": "native: no translation was found between the process and the CPU it runs on (no user-mode emulation, and an architecture the layers around it run natively); emulated: some layer translates the instructions. A system that its hypervisor emulates whole looks native from inside; see docs/CI-MATRIX.md \"CPU execution\".", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "enum": [ + "native", + "emulated", + null + ] + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_engine_mode": { + "description": "Whether the container engine runs without root on its host.", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "enum": [ + "rootless", + "rootful", + null + ] + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_seccomp": { + "description": "Seccomp mode of the process.", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "enum": [ + "disabled", + "strict", + "filter", + null + ] + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_selinux_mode": { + "description": "SELinux mode.", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "enum": [ + "enforcing", + "permissive", + "disabled", + null + ] + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_cgroup_version": { + "description": "cgroup hierarchy the process is in.", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "enum": [ + "v1", + "v2", + "hybrid", + null + ] + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "fact_escalation": { + "description": "Privilege escalation a stage used.", + "type": "object", + "required": [ + "value", + "provenance" + ], + "additionalProperties": false, + "properties": { + "value": { + "enum": [ + "none", + "sudo", + "root-login", + "setuid", + "other", + null + ] + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "reason": { + "$ref": "#/$defs/text" + } + } + }, + "os": { + "type": "object", + "required": [ + "family", + "name", + "version", + "kernel" + ], + "additionalProperties": false, + "properties": { + "family": { + "$ref": "#/$defs/fact_slug" + }, + "name": { + "$ref": "#/$defs/fact_slug" + }, + "version": { + "$ref": "#/$defs/fact_string" + }, + "kernel": { + "$ref": "#/$defs/fact_string" + } + } + }, + "cpu": { + "type": "object", + "required": [ + "arch", + "execution" + ], + "additionalProperties": false, + "properties": { + "arch": { + "$ref": "#/$defs/fact_slug" + }, + "execution": { + "$ref": "#/$defs/fact_execution" + } + } + }, + "service_manager": { + "type": "object", + "required": [ + "name", + "scope" + ], + "additionalProperties": false, + "properties": { + "name": { + "$ref": "#/$defs/fact_slug" + }, + "scope": { + "$ref": "#/$defs/fact_slug" + } + } + }, + "orchestration": { + "type": "object", + "required": [ + "name", + "version", + "scope" + ], + "additionalProperties": false, + "properties": { + "name": { + "$ref": "#/$defs/fact_slug" + }, + "version": { + "$ref": "#/$defs/fact_string" + }, + "scope": { + "$ref": "#/$defs/fact_slug" + } + } + }, + "container": { + "description": "Container-only facts; only a layer whose kind is container may have them.", + "type": "object", + "required": [ + "engine_mode", + "privileged", + "devices", + "orchestration" + ], + "additionalProperties": false, + "properties": { + "engine_mode": { + "$ref": "#/$defs/fact_engine_mode" + }, + "privileged": { + "$ref": "#/$defs/fact_boolean" + }, + "devices": { + "$ref": "#/$defs/fact_slug_list" + }, + "orchestration": { + "$ref": "#/$defs/orchestration" + } + } + }, + "layer": { + "description": "One execution layer. Layers are listed from the outermost (index 0) to the one the workload runs in (last).", + "type": "object", + "required": [ + "kind", + "technology", + "os", + "cpu", + "service_manager" + ], + "additionalProperties": false, + "properties": { + "label": { + "$ref": "#/$defs/text" + }, + "kind": { + "$ref": "#/$defs/fact_layer_kind" + }, + "technology": { + "$ref": "#/$defs/fact_slug" + }, + "os": { + "$ref": "#/$defs/os" + }, + "cpu": { + "$ref": "#/$defs/cpu" + }, + "service_manager": { + "$ref": "#/$defs/service_manager" + }, + "container": { + "$ref": "#/$defs/container" + }, + "extensions": { + "$ref": "#/$defs/extensions" + } + } + }, + "environment": { + "type": "object", + "required": [ + "layers" + ], + "additionalProperties": false, + "properties": { + "layers": { + "type": "array", + "items": { + "$ref": "#/$defs/layer" + }, + "minItems": 1, + "maxItems": 8 + } + } + }, + "python": { + "type": "object", + "required": [ + "implementation", + "version", + "cache_tag", + "soabi", + "platform", + "arch", + "pointer_bits", + "free_threaded_build", + "gil_enabled", + "libc", + "execution" + ], + "additionalProperties": false, + "properties": { + "implementation": { + "$ref": "#/$defs/fact_slug" + }, + "version": { + "$ref": "#/$defs/fact_string" + }, + "cache_tag": { + "$ref": "#/$defs/fact_string" + }, + "soabi": { + "$ref": "#/$defs/fact_string" + }, + "platform": { + "$ref": "#/$defs/fact_string" + }, + "arch": { + "$ref": "#/$defs/fact_slug" + }, + "pointer_bits": { + "$ref": "#/$defs/fact_integer" + }, + "free_threaded_build": { + "$ref": "#/$defs/fact_boolean" + }, + "gil_enabled": { + "$ref": "#/$defs/fact_boolean" + }, + "libc": { + "$ref": "#/$defs/fact_string" + }, + "execution": { + "$ref": "#/$defs/fact_execution" + } + } + }, + "node": { + "type": "object", + "required": [ + "version", + "modules_abi", + "napi_version", + "platform", + "arch", + "execution" + ], + "additionalProperties": false, + "properties": { + "version": { + "$ref": "#/$defs/fact_string" + }, + "modules_abi": { + "$ref": "#/$defs/fact_string" + }, + "napi_version": { + "$ref": "#/$defs/fact_string" + }, + "platform": { + "$ref": "#/$defs/fact_slug" + }, + "arch": { + "$ref": "#/$defs/fact_slug" + }, + "execution": { + "$ref": "#/$defs/fact_execution" + } + } + }, + "runtimes": { + "type": "object", + "required": [ + "python", + "node" + ], + "additionalProperties": false, + "properties": { + "python": { + "$ref": "#/$defs/python" + }, + "node": { + "$ref": "#/$defs/node" + } + } + }, + "identity": { + "description": "Who the process runs as. superuser: effective UID 0 on Linux (in the process's own user namespace), macOS and FreeBSD; on Windows, an elevated token, membership of Administrators, or SYSTEM. real_uid and saved_uid show whether a non-root process could switch back to UID 0; they are not-applicable on Windows.", + "type": "object", + "required": [ + "superuser", + "uid", + "real_uid", + "saved_uid", + "gid", + "user_namespace" + ], + "additionalProperties": false, + "properties": { + "superuser": { + "$ref": "#/$defs/fact_boolean" + }, + "uid": { + "$ref": "#/$defs/fact_integer" + }, + "real_uid": { + "$ref": "#/$defs/fact_integer" + }, + "saved_uid": { + "$ref": "#/$defs/fact_integer" + }, + "gid": { + "$ref": "#/$defs/fact_integer" + }, + "user_namespace": { + "type": "object", + "required": [ + "initial", + "uid_map" + ], + "additionalProperties": false, + "properties": { + "initial": { + "$ref": "#/$defs/fact_boolean" + }, + "uid_map": { + "$ref": "#/$defs/fact_uid_map" + } + } + } + } + }, + "isolation": { + "type": "object", + "required": [ + "no_new_privs", + "capabilities", + "seccomp", + "apparmor", + "selinux", + "cgroup", + "pid_namespace" + ], + "additionalProperties": false, + "properties": { + "no_new_privs": { + "$ref": "#/$defs/fact_boolean" + }, + "capabilities": { + "type": "object", + "required": [ + "effective", + "permitted", + "bounding" + ], + "additionalProperties": false, + "properties": { + "effective": { + "$ref": "#/$defs/fact_slug_list" + }, + "permitted": { + "$ref": "#/$defs/fact_slug_list" + }, + "bounding": { + "$ref": "#/$defs/fact_slug_list" + } + } + }, + "seccomp": { + "$ref": "#/$defs/fact_seccomp" + }, + "apparmor": { + "type": "object", + "required": [ + "enabled", + "profile" + ], + "additionalProperties": false, + "properties": { + "enabled": { + "$ref": "#/$defs/fact_boolean" + }, + "profile": { + "$ref": "#/$defs/fact_string" + } + } + }, + "selinux": { + "type": "object", + "required": [ + "mode", + "context" + ], + "additionalProperties": false, + "properties": { + "mode": { + "$ref": "#/$defs/fact_selinux_mode" + }, + "context": { + "$ref": "#/$defs/fact_string" + } + } + }, + "cgroup": { + "type": "object", + "required": [ + "version", + "delegated" + ], + "additionalProperties": false, + "properties": { + "version": { + "$ref": "#/$defs/fact_cgroup_version" + }, + "delegated": { + "$ref": "#/$defs/fact_boolean" + } + } + }, + "pid_namespace": { + "description": "Containers get their own PID namespace; the initial one shows the process is not in one.", + "type": "object", + "required": [ + "initial" + ], + "additionalProperties": false, + "properties": { + "initial": { + "$ref": "#/$defs/fact_boolean" + } + } + } + } + }, + "profile_workload": { + "type": "object", + "required": [ + "runtimes" + ], + "additionalProperties": false, + "properties": { + "runtimes": { + "$ref": "#/$defs/runtimes" + } + } + }, + "result_workload": { + "description": "The process that ran the collector, in the last layer.", + "type": "object", + "required": [ + "identity", + "isolation", + "runtimes" + ], + "additionalProperties": false, + "properties": { + "identity": { + "$ref": "#/$defs/identity" + }, + "isolation": { + "$ref": "#/$defs/isolation" + }, + "runtimes": { + "$ref": "#/$defs/runtimes" + } + } + }, + "backend": { + "type": "object", + "required": [ + "id", + "kind", + "api", + "available", + "device", + "version", + "features" + ], + "additionalProperties": false, + "properties": { + "id": { + "$ref": "#/$defs/slug" + }, + "kind": { + "enum": [ + "cpu", + "gpu", + "npu", + "other" + ] + }, + "api": { + "$ref": "#/$defs/slug" + }, + "available": { + "$ref": "#/$defs/fact_boolean" + }, + "device": { + "$ref": "#/$defs/fact_string" + }, + "version": { + "$ref": "#/$defs/fact_string" + }, + "features": { + "$ref": "#/$defs/fact_slug_list" + }, + "extensions": { + "$ref": "#/$defs/extensions" + } + } + }, + "compute": { + "type": "object", + "required": [ + "backends", + "accelerator_inventory" + ], + "additionalProperties": false, + "properties": { + "backends": { + "type": "array", + "items": { + "$ref": "#/$defs/backend" + }, + "minItems": 1, + "maxItems": 32 + }, + "accelerator_inventory": { + "$ref": "#/$defs/fact_boolean" + } + } + }, + "stage": { + "type": "object", + "required": [ + "status" + ], + "additionalProperties": false, + "properties": { + "status": { + "enum": [ + "passed", + "failed", + "blocked", + "not-run", + "not-applicable" + ] + }, + "reason": { + "$ref": "#/$defs/text" + }, + "evidence": { + "$ref": "#/$defs/evidence_list" + }, + "privilege": { + "type": "object", + "required": [ + "escalation" + ], + "additionalProperties": false, + "properties": { + "escalation": { + "$ref": "#/$defs/fact_escalation" + } + } + }, + "compute": { + "type": "object", + "required": [ + "requested", + "effective" + ], + "additionalProperties": false, + "properties": { + "requested": { + "$ref": "#/$defs/slug" + }, + "effective": { + "type": [ + "string", + "null" + ], + "pattern": "^[a-z0-9][a-z0-9._+-]{0,63}$" + } + } + }, + "extensions": { + "$ref": "#/$defs/extensions" + } + } + }, + "stages": { + "type": "object", + "required": [ + "detect", + "install", + "initialize", + "execute", + "correctness", + "security", + "performance" + ], + "additionalProperties": false, + "properties": { + "detect": { + "$ref": "#/$defs/stage" + }, + "install": { + "$ref": "#/$defs/stage" + }, + "initialize": { + "$ref": "#/$defs/stage" + }, + "execute": { + "$ref": "#/$defs/stage" + }, + "correctness": { + "$ref": "#/$defs/stage" + }, + "security": { + "$ref": "#/$defs/stage" + }, + "performance": { + "$ref": "#/$defs/stage" + } + } + }, + "ci": { + "type": "object", + "required": [ + "system", + "event", + "repository", + "ref", + "run_id", + "run_attempt", + "run_url", + "workflow_ref", + "job", + "runner_environment", + "runner_os", + "runner_arch", + "image_os", + "image_version" + ], + "additionalProperties": false, + "properties": { + "system": { + "$ref": "#/$defs/fact_slug" + }, + "event": { + "$ref": "#/$defs/fact_slug" + }, + "repository": { + "$ref": "#/$defs/fact_string" + }, + "ref": { + "$ref": "#/$defs/fact_string" + }, + "run_id": { + "$ref": "#/$defs/fact_string" + }, + "run_attempt": { + "$ref": "#/$defs/fact_integer" + }, + "run_url": { + "$ref": "#/$defs/fact_string" + }, + "workflow_ref": { + "$ref": "#/$defs/fact_string" + }, + "job": { + "$ref": "#/$defs/fact_string" + }, + "runner_environment": { + "$ref": "#/$defs/fact_slug" + }, + "runner_os": { + "$ref": "#/$defs/fact_string" + }, + "runner_arch": { + "$ref": "#/$defs/fact_string" + }, + "image_os": { + "$ref": "#/$defs/fact_string" + }, + "image_version": { + "$ref": "#/$defs/fact_string" + } + } + }, + "revision": { + "description": "tested_sha: the commit checked out and tested; event_sha: GITHUB_SHA, which for pull_request is GitHub's temporary merge commit; pr_head_sha and pr_base_sha: the pull request's head and base from the event payload.", + "type": "object", + "required": [ + "tested_sha", + "event_sha", + "pr_head_sha", + "pr_base_sha" + ], + "additionalProperties": false, + "properties": { + "tested_sha": { + "$ref": "#/$defs/fact_sha" + }, + "event_sha": { + "$ref": "#/$defs/fact_sha" + }, + "pr_head_sha": { + "$ref": "#/$defs/fact_sha" + }, + "pr_base_sha": { + "$ref": "#/$defs/fact_sha" + } + } + }, + "run": { + "type": "object", + "required": [ + "ci", + "revision" + ], + "additionalProperties": false, + "properties": { + "ci": { + "$ref": "#/$defs/ci" + }, + "revision": { + "$ref": "#/$defs/revision" + } + } + }, + "rule_id": { + "enum": [ + "app-non-root", + "engine-rootless", + "no-privileged-container", + "no-privilege-escalation", + "minimal-capabilities", + "minimal-devices" + ] + }, + "exception": { + "type": "object", + "required": [ + "id", + "rule", + "reason", + "scope", + "verification", + "end_condition", + "approval" + ], + "additionalProperties": false, + "properties": { + "id": { + "$ref": "#/$defs/slug" + }, + "rule": { + "$ref": "#/$defs/rule_id" + }, + "reason": { + "$ref": "#/$defs/text" + }, + "scope": { + "type": "object", + "required": [], + "additionalProperties": false, + "properties": { + "stages": { + "type": "array", + "items": { + "enum": [ + "detect", + "install", + "initialize", + "execute", + "correctness", + "security", + "performance" + ] + }, + "maxItems": 7 + }, + "escalations": { + "type": "array", + "items": { + "enum": [ + "sudo", + "root-login", + "setuid", + "other" + ] + }, + "maxItems": 4 + }, + "layers": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 7 + }, + "maxItems": 8 + }, + "capabilities": { + "type": "array", + "items": { + "$ref": "#/$defs/slug" + }, + "maxItems": 64 + }, + "devices": { + "type": "array", + "items": { + "$ref": "#/$defs/slug" + }, + "maxItems": 64 + } + } + }, + "verification": { + "$ref": "#/$defs/text" + }, + "end_condition": { + "$ref": "#/$defs/text" + }, + "approval": { + "type": "object", + "required": [ + "status" + ], + "additionalProperties": false, + "properties": { + "status": { + "enum": [ + "approved", + "pending", + "rejected" + ] + }, + "reference": { + "$ref": "#/$defs/text" + }, + "date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + } + } + } + } + }, + "policy": { + "type": "object", + "required": [], + "additionalProperties": false, + "properties": { + "rules": { + "type": "object", + "required": [], + "additionalProperties": false, + "properties": { + "app-non-root": { + "enum": [ + "required", + "preferred" + ] + }, + "engine-rootless": { + "enum": [ + "required", + "preferred" + ] + }, + "no-privileged-container": { + "enum": [ + "required", + "preferred" + ] + }, + "no-privilege-escalation": { + "enum": [ + "required", + "preferred" + ] + }, + "minimal-capabilities": { + "enum": [ + "required", + "preferred" + ] + }, + "minimal-devices": { + "enum": [ + "required", + "preferred" + ] + } + } + }, + "exceptions": { + "type": "array", + "items": { + "$ref": "#/$defs/exception" + }, + "maxItems": 32 + } + } + }, + "support": { + "description": "The lane's support status as docs/DEPENDENCY-STRATEGY.md records it; never derived from a run.", + "type": "object", + "required": [ + "status", + "lane", + "reference" + ], + "additionalProperties": false, + "properties": { + "status": { + "enum": [ + "checked", + "statically-resolvable", + "not-supported", + "not-assessed" + ] + }, + "lane": { + "type": [ + "string", + "null" + ], + "minLength": 1, + "maxLength": 200 + }, + "reference": { + "$ref": "#/$defs/text" + } + } + }, + "evidence_source": { + "type": "object", + "required": [ + "workflow", + "job" + ], + "additionalProperties": false, + "properties": { + "workflow": { + "$ref": "#/$defs/text" + }, + "job": { + "$ref": "#/$defs/slug" + } + } + }, + "runner": { + "description": "Run-level conditions a profile declares, compared with run.ci of each result checked against it.", + "type": "object", + "required": [], + "additionalProperties": false, + "properties": { + "runner_environment": { + "$ref": "#/$defs/fact_slug" + }, + "runner_os": { + "$ref": "#/$defs/fact_string" + }, + "runner_arch": { + "$ref": "#/$defs/fact_string" + }, + "image_os": { + "$ref": "#/$defs/fact_string" + } + } + }, + "profile": { + "description": "A declared lane: what its environment is meant to be and which policy applies. Not a measurement.", + "type": "object", + "required": [ + "schema_version", + "kind", + "synthetic", + "id", + "title", + "support", + "evidence_sources", + "required_stages", + "environment", + "workload", + "compute", + "policy" + ], + "additionalProperties": false, + "properties": { + "schema_version": { + "const": 1 + }, + "kind": { + "const": "profile" + }, + "synthetic": { + "type": "boolean" + }, + "id": { + "$ref": "#/$defs/slug" + }, + "title": { + "$ref": "#/$defs/text" + }, + "description": { + "$ref": "#/$defs/text" + }, + "support": { + "$ref": "#/$defs/support" + }, + "evidence_sources": { + "type": "array", + "items": { + "$ref": "#/$defs/evidence_source" + }, + "maxItems": 16 + }, + "runner": { + "$ref": "#/$defs/runner" + }, + "required_stages": { + "type": "array", + "items": { + "enum": [ + "detect", + "install", + "initialize", + "execute", + "correctness", + "security", + "performance" + ] + }, + "maxItems": 7 + }, + "accelerator_backend": { + "description": "The id of a gpu or npu backend in compute.backends that the execute, correctness and performance stages, when required, must run on. Absent or null for a CPU lane.", + "type": [ + "string", + "null" + ], + "pattern": "^[a-z0-9][a-z0-9._+-]{0,63}$" + }, + "environment": { + "$ref": "#/$defs/environment" + }, + "workload": { + "$ref": "#/$defs/profile_workload" + }, + "compute": { + "$ref": "#/$defs/compute" + }, + "policy": { + "$ref": "#/$defs/policy" + }, + "extensions": { + "$ref": "#/$defs/extensions" + } + } + }, + "result": { + "description": "One run on one lane: what was observed and what each stage did.", + "type": "object", + "required": [ + "schema_version", + "kind", + "synthetic", + "profile_id", + "generated_at", + "collector", + "run", + "environment", + "workload", + "compute", + "stages", + "overall", + "probe_errors" + ], + "additionalProperties": false, + "properties": { + "schema_version": { + "const": 1 + }, + "kind": { + "const": "result" + }, + "synthetic": { + "type": "boolean" + }, + "profile_id": { + "type": [ + "string", + "null" + ], + "pattern": "^[a-z0-9][a-z0-9._+-]{0,63}$" + }, + "generated_at": { + "type": [ + "string", + "null" + ], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + }, + "collector": { + "type": "object", + "required": [ + "name", + "version" + ], + "additionalProperties": false, + "properties": { + "name": { + "$ref": "#/$defs/slug" + }, + "version": { + "type": "integer", + "minimum": 1, + "maximum": 1000 + } + } + }, + "run": { + "$ref": "#/$defs/run" + }, + "environment": { + "$ref": "#/$defs/environment" + }, + "workload": { + "$ref": "#/$defs/result_workload" + }, + "compute": { + "$ref": "#/$defs/compute" + }, + "stages": { + "$ref": "#/$defs/stages" + }, + "overall": { + "enum": [ + "passed", + "failed", + "incomplete" + ] + }, + "probe_errors": { + "type": "array", + "maxItems": 50, + "items": { + "type": "object", + "required": [ + "probe", + "error" + ], + "additionalProperties": false, + "properties": { + "probe": { + "$ref": "#/$defs/slug" + }, + "error": { + "$ref": "#/$defs/text" + } + } + } + }, + "extensions": { + "$ref": "#/$defs/extensions" + } + } + } + } +}