diff --git a/.github/workflows/managed-resource-candidates.yml b/.github/workflows/managed-resource-candidates.yml new file mode 100644 index 0000000000..4e44da9b57 --- /dev/null +++ b/.github/workflows/managed-resource-candidates.yml @@ -0,0 +1,120 @@ +name: Managed resource candidates + +on: + pull_request: + paths: + - 'scripts/*managed_resource*' + - 'scripts/build_ops_bundle.py' + - 'scripts/ci_ops_bundle_smoke.py' + - 'scripts/build_intel_managed_wheel.py' + - 'scripts/managed_resource_intel_build.json' + - 'tests/test_managed_resource_producer.py' + - 'tests/test_ops_resources.py' + - 'src/powercontext/**' + - 'pyproject.toml' + - 'uv.lock' + - '.github/workflows/managed-resource-candidates.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: managed-resource-candidates-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + candidate: + name: Candidate ${{ matrix.target }} + strategy: + fail-fast: false + matrix: + include: + - target: windows-x86_64 + runner: windows-2025 + - target: macos-arm64 + runner: macos-15 + - target: macos-x86_64 + runner: macos-15-intel + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + defaults: + run: + shell: bash + env: + RESOURCE_TARGET: ${{ matrix.target }} + PACKAGE_SOURCE_SHA: ${{ github.sha }} + PACKAGE_VERSION: 0.0.0+candidate.${{ github.sha }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + persist-credentials: false + - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + with: + version: '0.13.5' + enable-cache: true + - name: Install locked build-host checks + run: uv sync --locked --no-dev --group resource-build --python 3.12 --no-build-package cryptography --no-build-package jieba + - name: Verify native source builders are not bootstrapped through Server extras + run: | + uv run --no-sync python - <<'PY' + import importlib.util + for module in ('cryptography', 'jieba', 'maturin'): + if importlib.util.find_spec(module) is not None: + raise RuntimeError(f'{module} must be acquired by pinned input preparation') + PY + - name: Select pinned Intel native compiler and SDK + if: matrix.target == 'macos-x86_64' + run: echo 'DEVELOPER_DIR=/Applications/Xcode_16.4.app/Contents/Developer' >> "$GITHUB_ENV" + - name: Check producer sources + run: | + uv run --no-sync ruff check --no-fix scripts/prepare_managed_resources.py scripts/produce_managed_resources.py scripts/ci_managed_resource_smoke.py scripts/build_intel_managed_wheel.py tests/test_managed_resource_producer.py + uv run --no-sync ruff format --check scripts/prepare_managed_resources.py scripts/produce_managed_resources.py scripts/ci_managed_resource_smoke.py scripts/build_intel_managed_wheel.py tests/test_managed_resource_producer.py + uv run --no-sync ty check scripts/prepare_managed_resources.py scripts/produce_managed_resources.py scripts/ci_managed_resource_smoke.py scripts/build_intel_managed_wheel.py tests/test_managed_resource_producer.py + - name: Acquire pinned interpreter and freeze native wheel closure + run: | + uv run --no-sync python scripts/prepare_managed_resources.py --target "$RESOURCE_TARGET" --source-ref "$PACKAGE_SOURCE_SHA" --server-version "$PACKAGE_VERSION" --output build/resource-inputs --download-cache build/resource-downloads + uv run --no-sync python - <<'PY' + import hashlib, os + from pathlib import Path + inputs = Path('build/resource-inputs').resolve() + digest = hashlib.sha256((inputs / 'inputs.json').read_bytes()).hexdigest() + with Path(os.environ['GITHUB_ENV']).open('a', encoding='utf-8') as output: + output.write(f'POWERCONTEXT_RESOURCE_TEST_INPUTS={inputs}\n') + output.write(f'POWERCONTEXT_RESOURCE_TEST_INPUTS_SHA256={digest}\n') + output.write(f'POWERCONTEXT_RESOURCE_TEST_OUTPUT={Path("build/resource-candidate").resolve()}\n') + output.write(f'POWERCONTEXT_RESOURCE_TEST_EVIDENCE={Path("build/resource-evidence").resolve()}\n') + PY + - name: Produce twice, compare bytes, consume catalog, and run bundled launcher/Ops + run: uv run --no-sync pytest -q tests/test_managed_resource_producer.py tests/test_ops_resources.py --junitxml build/resource-tests.xml + - name: Retain unqualified candidate evidence + if: success() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: unqualified-managed-resource-${{ matrix.target }}-${{ github.sha }} + path: | + build/resource-candidate/ + build/resource-inputs/inputs.json + build/resource-inputs/requirements.lock + build/resource-inputs/upstream-requirements.lock + build/resource-tests.xml + if-no-files-found: error + retention-days: 7 + - name: Record workflow diagnostics status + if: always() + run: | + mkdir -p build/resource-evidence + printf '%s\n' '{"status":"${{ job.status }}","target":"${{ matrix.target }}","qualification":"unqualified_candidate"}' > build/resource-evidence/workflow-status.json + - name: Retain bounded diagnostics and input identities + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: managed-resource-diagnostics-${{ matrix.target }}-${{ github.sha }} + path: | + build/resource-evidence/ + build/resource-tests.xml + build/resource-inputs/inputs.json + build/resource-inputs/requirements.lock + if-no-files-found: warn + retention-days: 7 diff --git a/docs/en/development/managed-resource-production.md b/docs/en/development/managed-resource-production.md new file mode 100644 index 0000000000..6abca00913 --- /dev/null +++ b/docs/en/development/managed-resource-production.md @@ -0,0 +1,208 @@ +--- +title: Managed runtime resource production +description: Build fixed, unqualified Server and independent Ops candidates for Windows x64 and both macOS architectures. +--- + +# Managed runtime resource production + +The candidate producer supplies the fixed resources consumed by [managed installation](../docs/operate/managed-local-server.md). +It produces a complete Server interpreter/package tree and a separate [Ops bundle](../docs/operate/local-server-ops.md). +Users of those artifacts need no system Python, uv, Git, or compiler. This is the resource-production part of +[#2112](https://github.com/oceanbase/powercontext/issues/2112); Desktop assembly and public distribution remain separate. + +Every output is an **unqualified candidate**. The scripts cannot promote a build by accepting a qualification flag. +Successful builds and CI smoke results do not establish signing, notarization, clean-machine operation, or native +Task Scheduler/LaunchAgent lifecycle acceptance. They do not publish artifacts or infer a production download URL. + +## Fixed inputs and supported build hosts + +Build natively on Windows x86_64, macOS arm64, or macOS x86_64. Linux is not a prerequisite and cross-compilation is +not supported. The build host needs Git, curl, uv 0.13.5, and the locked `resource-build` dependency group. This bootstrap +group imports the Ops resource/model contracts without Server transports, cryptography, jieba, or maturin. Installing +the full `server`/`ops` extras first would attempt an ambient Intel cryptography source build before the pinned builder. +Build-host tools +are not copied from that environment into either resource. + +`scripts/managed_resource_interpreters.json` pins CPython **3.12.16**, Astral python-build-standalone tag **20261026**, +and the exact install-only archive URL/SHA-256 for each target: + +| Target | Upstream artifact suffix | SHA-256 | +| --- | --- | --- | +| `windows-x86_64` | `x86_64-pc-windows-msvc-install_only.tar.gz` | `590dfcbe3a34c07301e6332e13e2188b98655cd072b08afa45a7998dc4c22bc5` | +| `macos-arm64` | `aarch64-apple-darwin-install_only.tar.gz` | `fbf4afef53c2a6b72bb741bc06d5671a9dfaf87bd84e5aa73ac9b3d0d589f14e` | +| `macos-x86_64` | `x86_64-apple-darwin-install_only.tar.gz` | `a962e47df522889eaa7ff0d6a56203f1079f68aabb65ffdbe8455f362bc769f3` | + +These are the upstream [20261026 release](https://github.com/astral-sh/python-build-standalone/releases/tag/20261026) +artifacts, using its documented [install tree](https://docs.astral.sh/python-build-standalone/distributions/). +The producer preserves in-tree interpreter and dependency notices and emits `resource-licenses.json` in both trees. +That index binds retained notices and wheel license metadata to the actual package hashes; it is not legal clearance. + +The preparation script takes an exact 40-character Git source commit and an explicit canonical package version. +It builds from `git archive`, so uncommitted files, editable installs, and the developer environment cannot enter the +project wheel. The supplied version is embedded with setuptools-scm's explicit version override and verified from +the resulting wheel and installed metadata. The source commit, archive, `uv.lock`, `pyproject.toml`, project wheel, +interpreter, build-tool closure, and all downloaded/built wheel identities are recorded in `inputs.json`. +The executing producer's Git revision and script digests are recorded separately from the packaged Server source. +`matches_committed_files=false` identifies a working-tree producer; it must not be described as a committed release builder. + +Runtime dependencies are exported from that source's locked `ops` and `cli` extras (`ops` includes `server`). Preparation +selects compatible wheels using the pinned interpreter's actual tags and environment markers, then verifies the +original `uv.lock` sizes and hashes. Only `jieba==0.42.1` may use a locked sdist: it has no upstream wheel. Its source +hash is checked before building with the exact hash-pinned tools in `scripts/managed_resource_build_tools.json`, +without build isolation or backend dependency downloads. The project wheel is the other explicit source build. +On Intel macOS, the exact locked `cryptography==50.0.0` is an additional source-build exception described below. +Other source-only dependencies fail clearly instead of enabling an implicit build. The resulting target wheel closure +is frozen as exact versions and individual wheel hashes in `requirements.lock`. + +Preparation may access the network. Final resource production uses only that verified wheelhouse, offline mode, +binary-only installation, and `--require-hashes`. Source builds are not claimed to be reproducible across arbitrary +toolchain/host changes: retain their actual wheel bytes and hashes. Repeated production from the same frozen inputs +must produce identical archives and manifests. + +### Intel cryptography source build + +Upstream cryptography [removed x86_64 macOS support in 49.0.0](https://cryptography.io/en/50.0.2/changelog/#v49-0-0). +The producer preserves the source lock's `50.0.0`, rather than downgrading to an older wheel. Its separate +`build_intel_managed_wheel.py` path builds the exact hashed sdist with these fixed native inputs: + +| Input | Fixed identity | +| --- | --- | +| Rust | `1.95.0`, `x86_64-apple-darwin` distribution SHA-256 `998af750fe2c28f0dbf193edc496c9a7407850486968f40ac11a8c8adfb7cfa6` | +| OpenSSL | `4.0.3` source SHA-256 `325b5c806167c13b40b1ffeadfe0248197c00eccc4cf123ec1e28d2d2fd216d9` | +| Backend | `maturin==1.14.1`, pinned Intel wheel; existing pinned setuptools plus native locked cffi/pycparser wheels | +| Compiler/SDK | Xcode `16.4` build `16F6`, macOS SDK `15.5`, deployment target `10.13` | + +All download URLs and hashes are in `scripts/managed_resource_intel_build.json`. Rust is installed only into a disposable +build directory. OpenSSL is built statically there; no Homebrew library is shipped or referenced. Every external Cargo +crate must have a crates.io checksum in the source's fixed `Cargo.lock`; `cargo vendor --locked` verifies acquisition, +then the build uses vendored dependencies and offline Cargo. Backend installation is hash-locked, offline, and binary-only. +Its bare `maturin` command resolves to the exact binary from the verified wheel in the task-local build environment. +Inherited target-prefixed OpenSSL settings and Python/backend/compiler overrides cannot redirect the pinned inputs. +The finished wheel retains OpenSSL and vendored Rust license notices. `otool` rejects non-system dynamic dependencies +and runtime search paths. Installed candidate probes perform AES-GCM roundtrip and Ed25519 signing/verification on all targets. +The source/tool/SDK identities and linkage results enter `inputs.json` before its digest is frozen. + +Intel build hosts must have that Xcode installed and select it for this process with +`DEVELOPER_DIR=/Applications/Xcode_16.4.app/Contents/Developer`; CI sets this variable without changing global Xcode selection. +This native build is a candidate for a platform upstream no longer supports. Compilation and smoke checks do not restore +upstream support or establish production maintenance/security qualification. The Intel CI leg must actually pass; it is +not skipped when a wheel is unavailable. No bit-for-bit source-build claim is made across different compiler/SDK identities. + +## Produce a candidate + +Use the current committed producer checkout, then choose the exact package-source commit and version. This executable +macOS arm64 recipe freezes the current commit and assigns an explicit candidate version: + +```bash +uv sync --locked --no-dev --group resource-build --python 3.12 --no-build-package cryptography --no-build-package jieba +package_source_sha="$(git rev-parse HEAD)" +uv run --no-sync python scripts/prepare_managed_resources.py \ + --target macos-arm64 \ + --source-ref "$package_source_sha" \ + --server-version "0.0.0+candidate.$package_source_sha" \ + --output build/resource-inputs \ + --download-cache build/resource-downloads +``` + +The supplied source commit must exist in this checkout and contain a compatible `uv.lock` and package build contract. +Use `macos-x86_64` on Intel macOS. The equivalent Windows PowerShell preparation is: + +```powershell +uv sync --locked --no-dev --group resource-build --python 3.12 --no-build-package cryptography --no-build-package jieba +$packageSourceSha = (git rev-parse HEAD).Trim() +uv run --no-sync python scripts/prepare_managed_resources.py --target windows-x86_64 --source-ref $packageSourceSha --server-version "0.0.0+candidate.$packageSourceSha" --output build/resource-inputs --download-cache build/resource-downloads +``` + +`--runtime-archive` can select an already downloaded interpreter archive; the same fixed hash +is still mandatory. The optional task-local download cache is digest-addressed, and every cache hit is rehashed. +An existing corrupt or incomplete input must be inspected before retrying; it is never silently adopted. + +Retain the exact `inputs_sha256` printed by preparation, then pass it to the offline producer: + +```bash +uv run --no-sync python scripts/produce_managed_resources.py \ + --inputs build/resource-inputs \ + --inputs-sha256 \ + --server-url https://candidate.invalid/server.tar.gz \ + --output build/resource-candidate +uv run --no-sync python scripts/ci_managed_resource_smoke.py \ + --candidate build/resource-candidate --serve --evidence-dir build/resource-evidence +``` + +`candidate.invalid` is a reserved test origin, not a usable download location. Supply an operator-controlled fixed HTTPS +candidate URL when testing real HTTP delivery; the producer neither uploads to it nor discovers one. The smoke tool +seeds the integrity-checked local cache with the exact archive, then uses real `ResourceCatalog` and `ResourceStore` +readback. It does not claim that an HTTP download occurred. + +If smoke fails, bounded fixture stdout/stderr and an exception-class summary are saved under the explicit evidence +directory before temporary cleanup. Each log is limited to its last 64 KiB; raw exception/child stdout is not printed +inline. Without `--evidence-dir`, the durable default is the candidate's sibling `-evidence` directory. + +Input and output directories must not contain one another. Existing outputs are never overwritten. Keep a failed or +partial build for inspection and retry at a fresh path. Keep large input/output/cache directories out of source control. + +## Output and launch contract + +| Output | Purpose | +| --- | --- | +| `server/server.tar.gz` | Deterministic, link-free Server tree, including its own Python and complete installed wheel closure. | +| `server/catalog.json`, `server/catalog.sha256` | Exact `ResourceCatalog`/`ResourceSpec` bytes and their digest. Each file has path, size, SHA-256, and executable mode. | +| `server/resource.json` | Complete source/input provenance plus the produced resource specification. | +| `ops/` | Independent `runtime/`, `packages/`, `entry.py`, and compatible `bundle.json`, built through `build_ops_bundle.py`. | +| `ops.tar.gz`, `ops-resource.json` | Deterministic Ops archive and complete file size/hash/executable inventory, including `bundle.json`. | +| `ops-provision.json` | Desktop distribution input pinning exact `ops/bundle.json` and `server/catalog.json` digests, with `binding_mode=managed_catalog`. | +| `candidate.json` | Unqualified status, source/package/output bindings, producer identities, and actual Windows path measurements when run on Windows. | + +The qualification reference starts with `unqualified-candidate:sha256:` and binds the frozen source/package inputs and +actual outputs. It provides no evidence of platform qualification. A checksum beside an untrusted artifact is not a +trust anchor. Desktop's packager must pin the descriptor in its trusted distribution and provision the independent +helper at the protected per-user `PowerContext/Ops/bundle` location, outside both the Desktop and managed Server trees. +Its catalog is packaged at `resource_dir/server/catalog.json`. Merely generating this directory does not install Desktop +or provision a helper. + +The Server runs the exact bundled interpreter with `-I -B -m powercontext.service.launcher`. Windows catalogs select +`pythonw.exe` and also inventory the paired `python.exe`; Ops uses the console interpreter. Ops retains the existing +`-I -S -B entry.py` launch contract and manifest format. Installer-generated console wrappers are removed because their +absolute build-path shebangs/launchers do not relocate. Mutable uv installer-cache records containing build-input +timestamps are also removed; distribution `RECORD` rows are repaired accordingly. Modules, +native libraries, wheel data, and license notices remain. Human CLI use can invoke the bundled callback with +` -I -B -c 'from powercontext.cli.app import main; main()' --help`. + +Interpreter archive links are never extracted as links. Safe relative aliases resolving to an internal regular file +are materialized as independent regular files and recorded in provenance. Directory aliases, external/absolute links, +cycles, special files, ambiguous paths, and reparse points are rejected. Gzip/tar metadata, ordering, timestamps, +permissions, and ownership are normalized; bytecode generation is disabled while probing sealed output trees. + +Windows production checks the actual current user's managed final directory, `staging/resource-<32 hex>` directory, +and independent Ops final directory against ordinary Win32 UTF-16 file/directory limits. It also invokes the consumer's +actual destination validator for Server paths. No system-wide long-path setting is enabled. Results on a short build +path do not qualify a different user profile; unsupported destination paths remain rejected by the consumer. + +## CI and evidence boundary + +`.github/workflows/managed-resource-candidates.yml` performs real native production on `windows-2025`, `macos-15` +(arm64), and `macos-15-intel`. Each matrix leg acquires pinned inputs from its exact commit, builds twice, compares +archive/manifest bytes after relocating the same hashed inputs, rejects mutation/version mismatches, consumes the real catalog/archive, and runs isolated +installed Server imports plus the exact service launcher against disposable SQLite data. The independent Ops smoke +starts with poisoned Python variables, audits module origins, and inspects a missing Server runtime with native +service-manager read-only commands. It never registers a service. Successful jobs retain explicitly unqualified candidates. +Bounded diagnostics, input identities and test evidence are retained on failure; partial runtimes and download caches are +not uploaded as failure diagnostics. A failed job is not a qualified resource. + +Run focused local checks with: + +```bash +uv run --no-sync pytest -q tests/test_managed_resource_producer.py tests/test_ops_resources.py +``` + +Without explicitly prepared native inputs, the expensive native producer tests skip. To enable the same native path, +set `POWERCONTEXT_RESOURCE_TEST_INPUTS` to the prepared directory and +`POWERCONTEXT_RESOURCE_TEST_INPUTS_SHA256` to its trusted `inputs.json` digest before running that command. +`POWERCONTEXT_RESOURCE_TEST_OUTPUT` optionally selects the first retained candidate directory; it must not exist. +Synthetic Windows archive/path tests verify rejection and layout logic, not actual Windows execution. + +Formal assembly/publication must still validate Windows signatures, macOS signing and notarization, clean standard-user +Windows 11/macOS operation without developer tools, login/reboot/service lifecycle, and preservation through repair and +removal. Signing can change interpreter/native-library bytes. Perform actual signing before freezing the hashes in any +qualified file inventory, archive, catalog, bundle manifest, or provision descriptor; regenerate and reverify the complete +identity chain afterward. Candidate digests must never be reused as proof of different signed artifacts. diff --git a/docs/en/development/meta.json b/docs/en/development/meta.json index 39bb09b4cd..5f20bc6663 100644 --- a/docs/en/development/meta.json +++ b/docs/en/development/meta.json @@ -4,6 +4,7 @@ "defaultOpen": false, "pages": [ "releasing", + "managed-resource-production", "core-protocol", "memory-layer", "plugin-contract", diff --git a/docs/en/docs/operate/local-server-ops.md b/docs/en/docs/operate/local-server-ops.md index b9946524be..d262829144 100644 --- a/docs/en/docs/operate/local-server-ops.md +++ b/docs/en/docs/operate/local-server-ops.md @@ -170,8 +170,9 @@ launch arguments, requirements digest, installed file bytes, and file digests. I make an untrusted runtime or wheel trustworthy. Existing output trees are never overwritten; inspect a partial build before removing it or building at a new location. -The release resource producer (#2112) supplies fixed input versions, hashes, platform/architecture identity, -redistribution licenses, and provenance. Windows/macOS packaging (#2108/#2109) must publish Ops to a stable, +The [candidate resource producer](../../development/managed-resource-production.md) supplies fixed input versions, +hashes, platform/architecture identity, retained license notices, and provenance for all three Windows/macOS targets. +Its output is always unqualified. Windows/macOS packaging (#2108/#2109) must publish Ops to a stable, user-protected location independent of both the Desktop installation and the managed Server. Desktop removal must not silently remove the maintenance tool or user data. Signing, macOS notarization, clean-machine execution, and native service/login/reboot acceptance are separate release checks; a successful bundle build does not establish them. diff --git a/docs/en/docs/operate/managed-local-server.md b/docs/en/docs/operate/managed-local-server.md index 1842df2e17..740ab34238 100644 --- a/docs/en/docs/operate/managed-local-server.md +++ b/docs/en/docs/operate/managed-local-server.md @@ -154,8 +154,10 @@ Scope ID, select an arbitrary Scope, or prove that a Memory operation has succee ## Release and platform acceptance -This implementation requires producer-supplied, fixed resources. The formal platform resource catalog and qualified -artifacts belong to [#2112](https://github.com/oceanbase/powercontext/issues/2112); the implementation does not supply a +This implementation requires producer-supplied, fixed resources. The +[candidate producer](../../development/managed-resource-production.md) builds pinned native Server/Ops resources for +Windows x86_64 and macOS arm64/x86_64 with real consumer readback. Its catalogs remain unqualified. +The formal platform resource catalog and qualified artifacts belong to [#2112](https://github.com/oceanbase/powercontext/issues/2112); the implementation does not supply a public production download URL or infer one from a version string. Development fixtures and locally built bundles verify implementation behavior, not production distribution trust. diff --git a/docs/zh/development/managed-resource-production.md b/docs/zh/development/managed-resource-production.md new file mode 100644 index 0000000000..665f8255a6 --- /dev/null +++ b/docs/zh/development/managed-resource-production.md @@ -0,0 +1,168 @@ +--- +title: 托管运行时资源生产 +description: 为 Windows x64、macOS arm64 与 x86_64 生成固定版本的 Server 和独立 Ops 候选资源。 +--- + +# 托管运行时资源生产 + +候选资源生产器为[托管安装](../docs/operate/managed-local-server.md)生成固定输入：完整的 Server Python 与依赖树，以及单独的 +[Ops 工具包](../docs/operate/local-server-ops.md)。最终用户不需要在系统中安装 Python、uv、Git 或编译器。这是 +[#2112](https://github.com/oceanbase/powercontext/issues/2112) 的资源生产部分；Desktop 安装包组装与公开发布另行完成。 + +所有输出都明确标记为**未资格验收的候选资源**。脚本不接受把构建直接标为生产合格的开关。构建与 CI 冒烟通过不代表完成了签名、 +公证、干净机器运行或 Task Scheduler/LaunchAgent 生命周期验收。脚本不会发布资源，也不会推测生产下载地址。 + +## 固定输入与构建主机 + +分别在 Windows x86_64、macOS arm64 和 macOS x86_64 上原生构建，不需要先完成 Linux 构建，也不支持交叉编译。构建主机需要 +Git、curl、uv 0.13.5 与锁定的 `resource-build` 依赖组。该引导环境可导入 Ops 资源/模型契约，不安装 Server 传输层、cryptography、 +jieba 或 maturin。先安装完整 `server`/`ops` extra 会在固定构建器运行前触发环境中的 Intel cryptography 源码构建。 +开发环境的工具和可编辑安装不会复制到资源中。 + +`scripts/managed_resource_interpreters.json` 固定 CPython **3.12.16**、Astral python-build-standalone 标签 **20261026**， +以及每个目标平台的完整 install-only 下载地址与 SHA-256： + +| 目标 | 上游产物后缀 | SHA-256 | +| --- | --- | --- | +| `windows-x86_64` | `x86_64-pc-windows-msvc-install_only.tar.gz` | `590dfcbe3a34c07301e6332e13e2188b98655cd072b08afa45a7998dc4c22bc5` | +| `macos-arm64` | `aarch64-apple-darwin-install_only.tar.gz` | `fbf4afef53c2a6b72bb741bc06d5671a9dfaf87bd84e5aa73ac9b3d0d589f14e` | +| `macos-x86_64` | `x86_64-apple-darwin-install_only.tar.gz` | `a962e47df522889eaa7ff0d6a56203f1079f68aabb65ffdbe8455f362bc769f3` | + +这些文件来自上游 [20261026 发布](https://github.com/astral-sh/python-build-standalone/releases/tag/20261026)，使用其文档定义的 +[安装目录树](https://docs.astral.sh/python-build-standalone/distributions/)。生产器保留解释器与各依赖在树内的许可说明，并在两个资源中 +生成 `resource-licenses.json`，记录声明、许可文件与实际 wheel 摘要的关系。该索引不是法律合规认证。 + +准备脚本要求精确的 40 位 Git 源提交与规范化包版本。项目 wheel 直接从 `git archive` 快照构建，未提交文件、可编辑安装和开发环境 +不会混入。构建时使用显式 setuptools-scm 版本覆盖，并核对生成的 wheel 与安装后元数据。`inputs.json` 记录源提交、源码归档、 +`uv.lock`、`pyproject.toml`、项目 wheel、解释器、构建工具闭包及所有下载/构建 wheel 的精确身份。执行生产器的 Git 提交与脚本摘要 +单独记录，不能与被打包的 Server 源提交混为一谈。`matches_committed_files=false` 表示执行了工作区中的生产器，不能称为已提交的发布构建器。 + +运行依赖从该源提交锁定的 `ops` 与 `cli` extra 导出；`ops` 已包含 `server`。准备阶段按固定解释器的真实 wheel 标签和环境标记选择 +兼容产物，再核对原始 `uv.lock` 中的大小和摘要。只有 `jieba==0.42.1` 可以使用锁定的 sdist，因为它没有上游 wheel。准备脚本先核对源码 +摘要，再用 `scripts/managed_resource_build_tools.json` 中完整固定版本与摘要的工具闭包构建，不启用构建隔离，也不允许构建后端额外下载依赖。 +项目 wheel 是另一个明确的源码构建；Intel macOS 还对精确锁定的 `cryptography==50.0.0` 使用下述专门源码构建路径。 +其他没有兼容 wheel 的依赖会明确失败，不会自动放开源码编译。 + +准备阶段可以联网。最终资源生产只安装冻结的 `requirements.lock` 与 wheelhouse，使用离线、仅 wheel 和强制摘要校验。固定工具输入并不 +保证任意主机/工具链上的源码构建都产生相同字节，因此必须保留实际 wheel 及摘要。相同冻结输入的重复资源生产必须生成相同归档和清单。 + +### Intel cryptography 源码构建 + +cryptography 上游从 [49.0.0 移除了 x86_64 macOS 支持](https://cryptography.io/en/50.0.2/changelog/#v49-0-0)。生产器保留源锁文件中的 +`50.0.0`，不降级使用旧 wheel。独立的 `build_intel_managed_wheel.py` 验证精确 sdist 后，使用以下固定原生输入构建： + +| 输入 | 固定身份 | +| --- | --- | +| Rust | `1.95.0`、`x86_64-apple-darwin` 分发包，SHA-256 `998af750fe2c28f0dbf193edc496c9a7407850486968f40ac11a8c8adfb7cfa6` | +| OpenSSL | `4.0.3` 源码，SHA-256 `325b5c806167c13b40b1ffeadfe0248197c00eccc4cf123ec1e28d2d2fd216d9` | +| 构建后端 | `maturin==1.14.1` 固定 Intel wheel、已有固定 setuptools，以及原生锁定的 cffi/pycparser wheel | +| 编译器/SDK | Xcode `16.4` build `16F6`、macOS SDK `15.5`、部署目标 `10.13` | + +所有下载地址与摘要固定在 `scripts/managed_resource_intel_build.json`。Rust 仅安装到一次性构建目录，OpenSSL 在该目录内静态构建， +资源不引用或携带 Homebrew 库。外部 Cargo crate 必须在固定源码的 `Cargo.lock` 中具有 crates.io 校验和，`cargo vendor --locked` +核对获取结果，随后使用 vendored 依赖和 Cargo 离线模式。后端仅从离线固定摘要 wheel 安装，其裸 `maturin` 命令必须解析至该 wheel +在任务构建环境中的精确二进制文件。继承的目标前缀 OpenSSL 设置及 Python/后端/编译器选项无法重定向固定输入。 +生成的 wheel 保留 OpenSSL 与 Rust 依赖许可文件。 +`otool` 拒绝非系统动态依赖及运行时搜索路径；三个平台安装后的探测都会实际执行 AES-GCM 往返和 Ed25519 签名/验签。 +源码、工具、SDK 身份和链接检查结果写入 `inputs.json` 后再冻结摘要。 + +Intel 构建主机须已安装该 Xcode，并为当前进程设置 `DEVELOPER_DIR=/Applications/Xcode_16.4.app/Contents/Developer`；CI 仅设置环境变量， +不修改系统全局 Xcode 选择。该构建针对上游已停止支持的平台，编译和冒烟通过不会恢复上游支持，也不能证明生产维护或安全资格。 +Intel CI 任务必须实际通过，不会因没有 wheel 而跳过；不同编译器/SDK 身份之间不宣称源码构建字节一致。 + +## 生成候选资源 + +使用当前已提交的生产器检出，再指定确切的包源码提交和版本。以下 macOS arm64 命令可直接执行，冻结当前提交并赋予明确的候选版本： + +```bash +uv sync --locked --no-dev --group resource-build --python 3.12 --no-build-package cryptography --no-build-package jieba +package_source_sha="$(git rev-parse HEAD)" +uv run --no-sync python scripts/prepare_managed_resources.py \ + --target macos-arm64 \ + --source-ref "$package_source_sha" \ + --server-version "0.0.0+candidate.$package_source_sha" \ + --output build/resource-inputs \ + --download-cache build/resource-downloads +``` + +源提交必须存在于当前检出中，且具有兼容的 `uv.lock` 和包构建契约。Intel macOS 使用 `macos-x86_64`。Windows PowerShell 对应命令： + +```powershell +uv sync --locked --no-dev --group resource-build --python 3.12 --no-build-package cryptography --no-build-package jieba +$packageSourceSha = (git rev-parse HEAD).Trim() +uv run --no-sync python scripts/prepare_managed_resources.py --target windows-x86_64 --source-ref $packageSourceSha --server-version "0.0.0+candidate.$packageSourceSha" --output build/resource-inputs --download-cache build/resource-downloads +``` + +`--runtime-archive` 可复用已经下载的解释器归档，仍必须满足同一个固定摘要。 +可选下载缓存位于显式指定的任务目录，按摘要寻址，每次命中都会重新校验。已经存在但损坏或未完成的输入必须先检查，不会直接复用。 + +保存准备脚本输出的 `inputs_sha256`，再传给离线生产器： + +```bash +uv run --no-sync python scripts/produce_managed_resources.py \ + --inputs build/resource-inputs \ + --inputs-sha256 <准备阶段输出的摘要> \ + --server-url https://candidate.invalid/server.tar.gz \ + --output build/resource-candidate +uv run --no-sync python scripts/ci_managed_resource_smoke.py \ + --candidate build/resource-candidate --serve --evidence-dir build/resource-evidence +``` + +`candidate.invalid` 是保留的测试域名，不能作为真实下载位置。验证 HTTP 交付时，应由操作者提供受控的固定 HTTPS 候选地址；生产器不负责 +上传或查找地址。冒烟工具把实际候选归档放入仍会校验摘要的本地缓存，再使用真实 `ResourceCatalog` 和 `ResourceStore` 回读，不宣称发生了 +HTTP 下载。输入与输出目录不能互相包含；已有输出绝不覆盖。失败或未完成的构建保留供检查，重试应使用新目录。大型输入、输出和缓存不可提交。 + +冒烟失败时，会在临时目录清理前，将有界的测试 stdout/stderr 和异常类型摘要保存到显式证据目录。每个日志最多保留末尾 64 KiB， +不在控制台内联打印原始异常内容或子进程 stdout。未指定 `--evidence-dir` 时，默认持久目录是候选目录旁的 `<候选目录名>-evidence`。 + +## 输出与启动契约 + +| 输出 | 用途 | +| --- | --- | +| `server/server.tar.gz` | 可重复、无链接的 Server 目录树，包含自身 Python 和完整安装依赖闭包。 | +| `server/catalog.json`、`server/catalog.sha256` | 精确的 `ResourceCatalog`/`ResourceSpec` 字节及摘要；清单逐文件记录路径、大小、SHA-256 和执行位。 | +| `server/resource.json` | 完整来源/输入溯源与生成的资源规格。 | +| `ops/` | 经现有 `build_ops_bundle.py` 构建的独立 `runtime/`、`packages/`、`entry.py` 与兼容的 `bundle.json`。 | +| `ops.tar.gz`、`ops-resource.json` | 可重复 Ops 归档及完整文件大小/摘要/执行位清单，包含 `bundle.json`。 | +| `ops-provision.json` | Desktop 分发输入，固定精确的 `ops/bundle.json` 和 `server/catalog.json` 摘要，使用 `binding_mode=managed_catalog`。 | +| `candidate.json` | 未验收状态、源码/包/输出绑定、生产器身份，以及在 Windows 上执行时的真实路径测量。 | + +资格引用以 `unqualified-candidate:sha256:` 开头，绑定冻结的源码/包输入与实际输出，不构成平台验收证据。跟随不可信产物提供的校验和不是 +信任根。Desktop 打包器须在受信任分发中固定描述文件，并把独立维护工具部署至受保护的当前用户 `PowerContext/Ops/bundle`，与 Desktop +安装和托管 Server 分离。catalog 打包位置为 `resource_dir/server/catalog.json`。生成上述目录本身不会安装 Desktop 或部署维护工具。 + +Server 用精确的内置解释器运行 `-I -B -m powercontext.service.launcher`。Windows catalog 选择 `pythonw.exe`，并同时清点配套的 +`python.exe`；Ops 使用控制台解释器，继续采用 `-I -S -B entry.py` 和原有清单格式。安装器生成的控制台包装器含绝对构建路径，无法迁移， +因此在冻结前删除。包含构建输入时间戳的可变 uv 安装器缓存记录也会移除，再同步修正发行版 `RECORD`。 +可导入模块、原生库、wheel 数据和许可文件保留；人工 CLI 可调用内置模块的 `main` 入口。 + +解释器归档中的链接不会直接解压为链接。仅将指向树内普通文件的安全相对别名物化为独立普通文件，并记录来源。目录别名、外部/绝对链接、 +循环、特殊文件、歧义路径和 reparse point 都被拒绝。归档顺序、时间戳、权限、所有者和 gzip/tar 元数据统一，验证冻结资源时禁用字节码生成。 + +Windows 生产会按普通 Win32 UTF-16 文件/目录上限核对当前用户真实的托管最终目录、`staging/resource-<32 位十六进制>` 目录及独立 Ops +最终目录。Server 还调用消费者实际使用的目标路径校验器，不启用系统级长路径设置。短构建路径通过不代表其他用户目录也合格；消费者仍拒绝 +不支持的真实安装路径。 + +## CI 与证据范围 + +`.github/workflows/managed-resource-candidates.yml` 在 `windows-2025`、`macos-15`（arm64）和 `macos-15-intel` 上实际生产。 +每个矩阵任务按精确源提交取得固定输入，迁移同一组摘要输入后构建两次，比较归档与清单字节、拒绝输出覆盖/版本不符，并通过真实 catalog/归档消费者回读。 +安装后的 Server 在隔离模式下导入依赖，通过精确 service launcher 启动一次性 SQLite Server 并验证就绪。独立 Ops 冒烟使用被污染的 +Python 环境变量，检查模块来源，并在 Server Python 缺失时通过原生服务管理器的只读命令检查状态，不注册任何服务。 +CI 在成功时保存明确标记为未资格验收的候选资源。失败时保留有界诊断、输入身份和测试证据，不把部分运行时或下载缓存作为失败诊断上传； +任务失败不能算合格资源。 + +本地聚焦检查： + +```bash +uv run --no-sync pytest -q tests/test_managed_resource_producer.py tests/test_ops_resources.py +``` + +没有显式准备的原生输入时，开销较大的原生生产测试跳过。将 `POWERCONTEXT_RESOURCE_TEST_INPUTS` 设置为准备目录，将 +`POWERCONTEXT_RESOURCE_TEST_INPUTS_SHA256` 设置为可信的 `inputs.json` 摘要，即可启用同一路径。 +`POWERCONTEXT_RESOURCE_TEST_OUTPUT` 可指定首个保留候选目录，该目录不能已经存在。合成 Windows 归档/路径测试仅证明布局与拒绝逻辑， +不能证明 Windows 上实际执行成功。 + +正式组装/发布仍需验证 Windows 签名、macOS 签名与公证、没有开发工具的干净标准用户 Windows 11/macOS 机器、登录/重启/服务生命周期， +以及修复和移除时保留配置与数据。签名会改变解释器或原生库字节。必须先完成真实签名，再冻结合格产物的文件清单、归档、catalog、bundle +清单和部署描述文件摘要，并重新生成、验证整个身份链。不得用未签名候选摘要证明另一组签名产物。 diff --git a/docs/zh/development/meta.json b/docs/zh/development/meta.json index 26d69af5ee..2157228f25 100644 --- a/docs/zh/development/meta.json +++ b/docs/zh/development/meta.json @@ -4,6 +4,7 @@ "defaultOpen": false, "pages": [ "releasing", + "managed-resource-production", "core-protocol", "memory-layer", "plugin-contract", diff --git a/docs/zh/docs/operate/local-server-ops.md b/docs/zh/docs/operate/local-server-ops.md index 34522b3539..84c3cbaa3e 100644 --- a/docs/zh/docs/operate/local-server-ops.md +++ b/docs/zh/docs/operate/local-server-ops.md @@ -141,7 +141,8 @@ macOS 使用对应的运行环境目录，并设置 `--python-relative bin/pytho 清单记录运行环境身份、启动参数、requirements 摘要、安装文件字节数和文件摘要。它是产物清单，不是信任锚或签名，不能把不可信运行环境或 wheel 变成可信输入。 已有输出目录不会被覆盖，构建失败后应检查残留产物，再决定清理或换目录构建。 -资源生产方（#2112）负责固定输入版本、摘要、平台与架构身份、再分发许可和来源信息。 +[候选资源生产器](../../development/managed-resource-production.md)为 Windows/macOS 三个目标固定输入版本、摘要、平台与架构身份， +保留许可说明并记录来源；输出始终是未资格验收的候选资源。 Windows/macOS 打包（#2108/#2109）需要将 Ops 发布到受用户权限保护、独立于 Desktop 和受管 Server 的稳定目录。 卸载 Desktop 不应悄悄删除维护工具或用户数据。 签名、macOS notarization、干净机器执行以及原生服务、登录和重启验收是单独的发布检查，构建成功不能代替这些证据。 diff --git a/docs/zh/docs/operate/managed-local-server.md b/docs/zh/docs/operate/managed-local-server.md index c966e0d718..688c181b82 100644 --- a/docs/zh/docs/operate/managed-local-server.md +++ b/docs/zh/docs/operate/managed-local-server.md @@ -86,7 +86,10 @@ description: 安装或修复固定版本的个人 Server 运行环境，并保 ## 发布与平台验收 -实现要求生产者提供固定资源。正式平台资源目录及合格产物归 [#2112](https://github.com/oceanbase/powercontext/issues/2112) 负责；实现不提供公共生产下载 URL，也不会根据版本号猜测地址。开发 fixture 和本地构建包验证的是实现行为，不能证明生产分发可信。 +实现要求生产者提供固定资源。[候选生产器](../../development/managed-resource-production.md)为 Windows x86_64 和 macOS arm64/x86_64 +原生构建固定的 Server/Ops 资源，并通过真实消费者回读；其 catalog 仍未资格验收。正式平台资源目录及合格产物归 +[#2112](https://github.com/oceanbase/powercontext/issues/2112) 负责；实现不提供公共生产下载 URL，也不会根据版本号猜测地址。 +开发 fixture 和本地构建包验证的是实现行为，不能证明生产分发可信。 Windows 预检会根据实际最终目录和固定暂存路径预算检查资源布局，获取资源时还会在下载前核对本次暂存目录。普通路径必须满足 Windows 文件和目录长度限制，不支持的布局返回 `resource_path_unsupported`。Ops 不修改系统长路径设置，也不假定 Task Scheduler 支持扩展路径形式的可执行文件名。生产者需提供适合目标用户目录的合格布局；完整开发包在较短测试目录中启动成功，不能证明默认安装布局可用。 diff --git a/e2e/bub/uv.lock b/e2e/bub/uv.lock index 730fc0e4c0..b9a77a0b5d 100644 --- a/e2e/bub/uv.lock +++ b/e2e/bub/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12, <4.0" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -1823,6 +1823,30 @@ notebooks = [ { name = "powercontext-langgraph", editable = "../../integrations/langgraph" }, { name = "python-dotenv", specifier = ">=1,<2" }, ] +resource-build = [ + { name = "aiosqlite", specifier = ">=0.22,<1" }, + { name = "alembic", specifier = ">=1.16,<2" }, + { name = "apscheduler", specifier = ">=3.11,<4" }, + { name = "click", specifier = ">=8.1,<9" }, + { name = "filelock", specifier = ">=3.16,<4" }, + { name = "httpx", specifier = ">=0.28,<1" }, + { name = "jsonschema", specifier = ">=4.23,<5" }, + { name = "opentelemetry-api", specifier = ">=1.30,<2" }, + { name = "opentelemetry-sdk", specifier = ">=1.30,<2" }, + { name = "packaging", specifier = ">=24,<27" }, + { name = "platformdirs", specifier = ">=4,<5" }, + { name = "pydantic-settings", specifier = ">=2.7,<3" }, + { name = "pyobvector", specifier = ">=0.2.28,<0.3" }, + { name = "pytest", specifier = ">=9.0.2" }, + { name = "pyyaml", specifier = ">=6,<7" }, + { name = "rich", specifier = ">=13,<16" }, + { name = "ruff", specifier = ">=0.15.7" }, + { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2,<3" }, + { name = "sqlite-vec", specifier = ">=0.1.9,<0.2" }, + { name = "ty", specifier = ">=0.0.24" }, + { name = "tzdata", specifier = ">=2026.3" }, + { name = "zstandard", specifier = ">=0.23,<1" }, +] [[package]] name = "powercontext-bub" diff --git a/pyproject.toml b/pyproject.toml index 2a878fec9a..8d5424d13b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,32 @@ Repository = "https://github.com/oceanbase/powercontext" Documentation = "https://powercontext.oceanbase.io/en/docs/" [dependency-groups] +# Producer/consumer checks import Ops settings and models, without installing +# Server transports or cryptography before the pinned native builder runs. +resource-build = [ + "aiosqlite>=0.22,<1", + "alembic>=1.16,<2", + "apscheduler>=3.11,<4", + "click>=8.1,<9", + "filelock>=3.16,<4", + "httpx>=0.28,<1", + "jsonschema>=4.23,<5", + "opentelemetry-api>=1.30,<2", + "opentelemetry-sdk>=1.30,<2", + "packaging>=24,<27", + "platformdirs>=4,<5", + "pydantic-settings>=2.7,<3", + "pyobvector>=0.2.28,<0.3", + "pyyaml>=6,<7", + "rich>=13,<16", + "sqlalchemy[asyncio]>=2,<3", + "sqlite-vec>=0.1.9,<0.2", + "tzdata>=2026.3", + "zstandard>=0.23,<1", + "pytest>=9.0.2", + "ruff>=0.15.7", + "ty>=0.0.24", +] notebooks = [ "powercontext[cli,server]", "powercontext-langchain", diff --git a/scripts/build_intel_managed_wheel.py b/scripts/build_intel_managed_wheel.py new file mode 100644 index 0000000000..c607fdfbd3 --- /dev/null +++ b/scripts/build_intel_managed_wheel.py @@ -0,0 +1,390 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build locked cryptography for Intel macOS with local, static OpenSSL. + +Invoked only by input preparation. Upstream removed this platform, so producing +and smoke-testing this wheel cannot claim upstream support or qualification. +""" + +# ruff: noqa: TRY003, S603 - fixed native producer toolchain and pinned source builds + +from __future__ import annotations + +import hashlib +import json +import platform +import re +import shutil +import subprocess +import tarfile +import tempfile +import tomllib +import zipfile +from pathlib import Path +from typing import Any + +PINS = Path(__file__).with_name("managed_resource_intel_build.json") + + +def _hash(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def _unpack(path: Path, destination: Path) -> Path: + destination.mkdir() + with tarfile.open(path) as archive: + archive.extractall(destination, filter="data") + roots = list(destination.iterdir()) + if len(roots) != 1 or not roots[0].is_dir(): + raise ValueError("Native build source must have exactly one root directory") + return roots[0] + + +def _retain_native_licenses(tree: Path, vendor: Path, openssl: Path, crates: list[dict[str, Any]]) -> None: + licenses = next(tree.glob("*.dist-info")) / "licenses" + licenses.mkdir(exist_ok=True) + (licenses / "OPENSSL-LICENSE.txt").write_bytes((openssl / "LICENSE.txt").read_bytes()) + records = [] + for crate in crates: + root = vendor / f"{crate['name']}-{crate['version']}" + package = tomllib.loads((root / "Cargo.toml").read_text(encoding="utf-8"))["package"] + notices = [] + for path in sorted(root.rglob("*")): + relative = path.relative_to(root) + if path.is_file() and any( + part.lower().startswith(("license", "copying", "notice")) for part in relative.parts + ): + if path.is_symlink() or not path.resolve().is_relative_to(root): + raise ValueError("Vendored license notices must be regular in-tree files") + destination = licenses / "rust" / root.name / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, destination) + notices.append({"path": destination.relative_to(tree).as_posix(), "sha256": _hash(destination)}) + records.append({ + "name": crate["name"], + "version": crate["version"], + "source_sha256": crate["checksum"], + "license": package.get("license"), + "notices": notices, + }) + (licenses / "native-build-licenses.json").write_text( + json.dumps({"format_version": 1, "crates": records}, ensure_ascii=False, sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + + +def _check_linkage(tree: Path, env: dict[str, str]) -> list[dict[str, Any]]: + linkage = [] + for library in tree.rglob("*.so"): + commands = subprocess.check_output(["/usr/bin/otool", "-l", str(library)], env=env, text=True) + dependencies, rpaths, install_names = [], [], [] + for block in re.split(r"\nLoad command \d+\n", commands): + command = re.search(r"^\s*cmd (LC_\w+)$", block, re.MULTILINE) + if command is None: + continue + kind = command[1] + if kind == "LC_RPATH": + field, values = "path", rpaths + elif kind.endswith("_DYLIB"): + # otool -L also lists LC_ID_DYLIB: the extension's own identity + # is not a dependency loaded from an external runtime directory. + field, values = "name", install_names if kind == "LC_ID_DYLIB" else dependencies + else: + continue + value = re.search(rf"^\s*{field} (.*?) \(offset \d+\)$", block, re.MULTILINE) + if value is None: + raise ValueError("Native extension has an unreadable dynamic-library load command") + values.append(value[1]) + if any(not item.startswith(("/usr/lib/", "/System/Library/")) for item in [*dependencies, *rpaths]): + raise ValueError("Native cryptography wheel depends on non-system libraries or runtime search paths") + linkage.append({ + "path": library.relative_to(tree).as_posix(), + "install_names": install_names, + "dependencies": dependencies, + "rpaths": rpaths, + }) + if not linkage: + raise ValueError("Native cryptography wheel contains no checked native extension") + return linkage + + +def native_build_environment( + base: dict[str, str], *, python: Path, rust: Path, workspace: Path, compiler: str, sdk_path: str, deployment: str +) -> dict[str, str]: + """Select only the fixed backend/interpreter/compiler and native library inputs.""" + # Start from operational necessities, not a growing list of compiler knobs: + # prefixed OpenSSL/cc variables and Conda/virtualenv controls must not win. + allowed = { + "HOME", + "USER", + "LOGNAME", + "TMPDIR", + "TMP", + "TEMP", + "LANG", + "LC_ALL", + "DEVELOPER_DIR", + "SOURCE_DATE_EPOCH", + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "NO_PROXY", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + } + clean = {key: value for key, value in base.items() if key.upper() in allowed or key.upper().startswith("LC_")} + return { + **clean, + "PATH": str(python.parent) + ":" + str(rust / "bin") + ":/usr/bin:/bin:/usr/sbin:/sbin", + "SDKROOT": sdk_path, + "CC": compiler, + "MACOSX_DEPLOYMENT_TARGET": deployment, + "CARGO_HOME": str(workspace / "cargo-home"), + "RUSTUP_HOME": str(workspace / "unused-rustup"), + "RUSTC": str(rust / "bin/rustc"), + } + + +def verify_backend(archive: Path, python: Path, environment: dict[str, str]) -> str: + """A bare PEP 517 maturin command must resolve to the exact pinned wheel binary.""" + executable = python.parent / "maturin" + selected = shutil.which("maturin", path=environment["PATH"]) + with zipfile.ZipFile(archive) as wheel: + names = [name for name in wheel.namelist() if name.endswith(".data/scripts/maturin")] + if len(names) != 1: + raise ValueError("Pinned maturin wheel must contain one backend executable") + expected = hashlib.sha256(wheel.read(names[0])).hexdigest() + if selected is None or Path(selected) != executable or _hash(executable) != expected: + raise ValueError("The backend command does not resolve to the pinned maturin executable") + return expected + + +def build( # noqa: C901 - ordered native build and verification boundary + *, + source_archive: Path, + source_identity: dict[str, Any], + artifacts: Path, + dependencies: list[dict[str, Any]], + python: Path, + uv: str, + wheels: Path, + environment: dict[str, str], +) -> dict[str, Any]: + pins = json.loads(PINS.read_bytes()) + if platform.system() != "Darwin" or platform.machine().lower() != "x86_64": + raise ValueError("The Intel cryptography source builder requires native Intel macOS") + if (source_identity["name"], source_identity["version"]) != (pins["package"], pins["version"]): + raise ValueError("The Intel source-build exception applies only to the explicitly pinned package/version") + if _hash(source_archive) != source_identity["sha256"]: + raise ValueError("Native package source changed after acquisition") + for item in pins["artifacts"].values(): + path = artifacts / item["filename"] + if _hash(path) != item["sha256"]: + raise ValueError("Native build-tool/source digest mismatch") + xcode = subprocess.check_output(["/usr/bin/xcodebuild", "-version"], env=environment, text=True).strip() + sdk = subprocess.check_output( + ["/usr/bin/xcrun", "--sdk", "macosx", "--show-sdk-version"], env=environment, text=True + ).strip() + if xcode != f"Xcode {pins['xcode_version']}\nBuild version {pins['xcode_build']}" or sdk != pins["sdk_version"]: + raise ValueError("Intel source production requires the pinned Xcode build and macOS SDK") + sdk_path = subprocess.check_output( + ["/usr/bin/xcrun", "--sdk", "macosx", "--show-sdk-path"], env=environment, text=True + ).strip() + compiler = subprocess.check_output(["/usr/bin/xcrun", "--find", "clang"], env=environment, text=True).strip() + with tempfile.TemporaryDirectory(prefix="pc-intel-wheel-") as temporary: + work = Path(temporary).resolve() + source = _unpack(source_archive, work / "package") + openssl = _unpack(artifacts / pins["artifacts"]["openssl"]["filename"], work / "openssl-source") + rust_source = _unpack(artifacts / pins["artifacts"]["rust"]["filename"], work / "rust-source") + rust = work / "rust" + env = native_build_environment( + environment, + python=python, + rust=rust, + workspace=work, + compiler=compiler, + sdk_path=sdk_path, + deployment=pins["deployment_target"], + ) + subprocess.run( + [ + "/bin/sh", + str(rust_source / "install.sh"), + "--prefix=" + str(rust), + "--disable-ldconfig", + "--components=rustc,cargo,rust-std-x86_64-apple-darwin", + ], + check=True, + env=env, + cwd=work, + timeout=300, + ) + rust_version = subprocess.check_output([str(rust / "bin/rustc"), "--version"], env=env, text=True).strip() + if not rust_version.startswith("rustc 1.95.0 "): + raise ValueError("Pinned Rust distribution reports an unexpected version") + prefix = work / "openssl" + subprocess.run( + [ + "/usr/bin/perl", + "Configure", + "darwin64-x86_64-cc", + "no-shared", + "no-module", + "no-tests", + "-fPIC", + "--prefix=" + str(prefix), + "--libdir=lib", + ], + cwd=openssl, + env=env, + check=True, + timeout=120, + ) + subprocess.run(["/usr/bin/make", "-j2"], cwd=openssl, env=env, check=True, timeout=1800) + subprocess.run(["/usr/bin/make", "install_sw"], cwd=openssl, env=env, check=True, timeout=300) + openssl_version = subprocess.check_output([str(prefix / "bin/openssl"), "version"], env=env, text=True).strip() + if not openssl_version.startswith("OpenSSL 4.0.3 "): + raise ValueError("Pinned OpenSSL source reports an unexpected version") + maturin = pins["artifacts"]["maturin"] + build_requirements = work / "backend-requirements.lock" + closure = [item for item in dependencies if item["name"] in {"cffi", "pycparser"}] + if {item["name"] for item in closure} != {"cffi", "pycparser"}: + raise ValueError("Intel native backend requires the complete locked cffi/pycparser closure") + build_requirements.write_text( + "maturin==1.14.1 --hash=sha256:" + + maturin["sha256"] + + "\n" + + "".join(f"{p['name']}=={p['version']} --hash=sha256:{p['sha256']}\n" for p in closure), + encoding="utf-8", + ) + subprocess.run( + [ + uv, + "--offline", + "pip", + "install", + "--python", + str(python), + "--no-index", + "--find-links", + str(artifacts), + "--only-binary", + ":all:", + "--require-hashes", + "-r", + str(build_requirements), + ], + check=True, + env=env, + timeout=120, + ) + backend_digest = verify_backend(artifacts / maturin["filename"], python, env) + lock_path = source / "Cargo.lock" + lock = tomllib.loads(lock_path.read_text(encoding="utf-8")) + crates = [item for item in lock["package"] if "source" in item] + if any( + item["source"] != "registry+https://github.com/rust-lang/crates.io-index" + or re.fullmatch(r"[0-9a-f]{64}", item.get("checksum", "")) is None + for item in crates + ): + raise ValueError("All external Cargo dependencies must have locked crates.io checksums") + lock_digest = _hash(lock_path) + cargo = rust / "bin/cargo" + vendored = subprocess.check_output( + [ + str(cargo), + "vendor", + "--locked", + "--versioned-dirs", + "--manifest-path", + str(source / "Cargo.toml"), + str(work / "vendor"), + ], + cwd=source, + env=env, + text=True, + timeout=1800, + ) + (source / ".cargo").mkdir(exist_ok=True) + (source / ".cargo/config.toml").write_text(vendored, encoding="utf-8") + env.update( + CARGO_NET_OFFLINE="true", + CARGO_TARGET_DIR=str(work / "target"), + OPENSSL_DIR=str(prefix), + OPENSSL_STATIC="1", + OPENSSL_NO_VENDOR="1", + PYO3_PYTHON=str(python), + ) + subprocess.run( + [ + str(python), + "-I", + "-B", + "-m", + "pip", + "--no-cache-dir", + "wheel", + "--no-deps", + "--no-build-isolation", + "--no-index", + "--wheel-dir", + str(wheels), + str(source), + ], + cwd=source, + env=env, + check=True, + timeout=1800, + ) + if _hash(lock_path) != lock_digest: + raise ValueError("The source build changed Cargo.lock") + produced = list(wheels.glob("cryptography-50.0.0-*.whl")) + if len(produced) != 1: + raise ValueError("Native source build must produce one exact cryptography wheel") + unpacked = work / "wheel" + subprocess.run( + [str(python), "-I", "-B", "-m", "wheel", "unpack", str(produced[0]), "-d", str(unpacked)], + check=True, + env=env, + timeout=60, + ) + tree = next(unpacked.iterdir()) + _retain_native_licenses(tree, work / "vendor", openssl, crates) + linkage = _check_linkage(tree, env) + produced[0].unlink() + subprocess.run( + [str(python), "-I", "-B", "-m", "wheel", "pack", str(tree), "-d", str(wheels)], + check=True, + env=env, + timeout=60, + ) + return { + "status": "unqualified_native_source_build", + "source_sha256": source_identity["sha256"], + "pins_sha256": _hash(PINS), + "pins": pins, + "cargo_lock_sha256": lock_digest, + "crates": [ + {"name": item["name"], "version": item["version"], "sha256": item["checksum"]} for item in crates + ], + "rust_version": rust_version, + "maturin_executable_sha256": backend_digest, + "openssl_version": openssl_version, + "xcode": xcode, + "sdk_version": sdk, + "compiler": subprocess.check_output([compiler, "--version"], env=env, text=True).splitlines()[0], + "linkage": linkage, + } diff --git a/scripts/ci_managed_resource_smoke.py b/scripts/ci_managed_resource_smoke.py new file mode 100644 index 0000000000..316b451bad --- /dev/null +++ b/scripts/ci_managed_resource_smoke.py @@ -0,0 +1,209 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read back a native candidate through the real managed-resource consumer. + +The cache is seeded with the exact candidate archive; this exercises archive, +catalog, relocation and installed imports without claiming a public download or +native service-manager/clean-machine qualification. +""" + +# ruff: noqa: TRY003, S603 - fixed local artifact checks, no service registration + +from __future__ import annotations + +import argparse +import dataclasses +import json +import os +import shutil +import socket +import subprocess +import tempfile +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from uuid import uuid4 + +import httpx +from ci_ops_bundle_smoke import check as check_ops +from prepare_managed_resources import environment, sha256 +from produce_managed_resources import _server_probe + +from powercontext.ops.local_policy import new_configuration +from powercontext.ops.resources import ResourceCatalog, ResourceSpec, ResourceStore, verify +from powercontext.service.environment import protect_directory, write_protected_file + +MAX_DIAGNOSTIC_BYTES = 64 * 1024 + + +@contextmanager +def preserve_failure(evidence: Path, workspace: Path) -> Iterator[None]: + """Keep bounded fixture logs before temporary workspace cleanup; suppress raw errors.""" + try: + yield + except Exception as error: + saved = evidence / ("failure-" + uuid4().hex) + saved.mkdir(parents=True) + captured = [] + for name in ("stdout.log", "stderr.log"): + source = workspace / "control" / name + if source.is_file() and not source.is_symlink(): + with source.open("rb") as stream: + stream.seek(max(0, source.stat().st_size - MAX_DIAGNOSTIC_BYTES)) + (saved / name).write_bytes(stream.read(MAX_DIAGNOSTIC_BYTES)) + captured.append(name) + if isinstance(error, (subprocess.CalledProcessError, subprocess.TimeoutExpired)) and error.stderr: + content = error.stderr.encode("utf-8") if isinstance(error.stderr, str) else error.stderr + (saved / "subprocess-stderr.log").write_bytes(content[-MAX_DIAGNOSTIC_BYTES:]) + captured.append("subprocess-stderr.log") + (saved / "failure.json").write_text( + json.dumps( + {"status": "candidate_smoke_failed", "exception_class": type(error).__name__, "fixture_logs": captured}, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + raise RuntimeError(f"Candidate smoke failed; bounded diagnostics: {saved}") from None + + +def _serve(root: Path, spec: ResourceSpec, working: Path, env: dict[str, str]) -> dict[str, object]: + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + data = working / "data" + endpoint = f"http://127.0.0.1:{port}" + configuration = new_configuration(data, port) + identity = write_protected_file(working / "server.env", configuration) + arguments = [ + str(root / spec.python_relative), + "-I", + "-B", + "-m", + "powercontext.service.launcher", + "--endpoint", + endpoint, + "--data-dir", + str(data), + "--env-file", + str(working / "server.env"), + "--stdout", + str(working / "stdout.log"), + "--stderr", + str(working / "stderr.log"), + ] + for name, value in dataclasses.asdict(identity).items(): + if name != "path" and value is not None: + arguments.extend(["--env-file-" + name.replace("_", "-"), str(value)]) + with (working / "stdout.log").open("wb") as stdout, (working / "stderr.log").open("wb") as stderr: + process = subprocess.Popen(arguments, cwd=working, env=env, stdout=stdout, stderr=stderr) + try: + deadline = time.monotonic() + 120 + with httpx.Client(trust_env=False, timeout=2) as client: + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("Bundled service launcher exited before readiness; inspect the smoke logs") + try: + response = client.get(endpoint + "/health/ready") + except httpx.TransportError: + time.sleep(0.2) + continue + if response.status_code == 200: + return {"launcher": "passed", "readiness_status": response.json()["status"]} + time.sleep(0.2) + raise RuntimeError("Bundled service launcher did not become ready within the smoke budget") + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + + +def check(candidate: Path, *, serve: bool = False, evidence_dir: Path | None = None) -> dict[str, object]: + candidate = candidate.resolve() + evidence = evidence_dir.resolve() if evidence_dir else candidate.with_name(candidate.name + "-evidence") + with ( + tempfile.TemporaryDirectory(prefix="pc-candidate-readback-") as temporary, + preserve_failure(evidence, Path(temporary).resolve()), + ): + work = Path(temporary).resolve() + catalog_digest = (candidate / "server/catalog.sha256").read_text(encoding="utf-8").strip() + control = protect_directory(work / "control", create=True) + path = control / "catalog.json" + write_protected_file(path, (candidate / "server/catalog.json").read_bytes()) + catalog = ResourceCatalog.read(path, catalog_digest) + spec = catalog.select_current() + cache = protect_directory(work / "cache", create=True) + shutil.copyfile(candidate / "server/server.tar.gz", cache / (spec.archive_sha256 + ".archive")) + prepared = ResourceStore(cache).prepare(spec, work / "staging") + verify(spec, prepared.directory) + moved = work / "relocated server" + prepared.directory.rename(moved) + if os.name != "nt": + moved.chmod(0o700) + spec_values = { + "platform": spec.platform, + "architecture": spec.architecture, + "python_version": spec.python_version, + "python_relative": spec.python_console_relative, + "server_python_relative": spec.python_relative, + } + poison = work / "poison" + poison.mkdir() + (poison / "powercontext.py").write_text("raise RuntimeError('checkout import leaked')\n", encoding="utf-8") + env = {**environment(), "PATH": "", "PYTHONPATH": str(poison), "PYTHONHOME": str(poison / "missing")} + _server_probe(moved, spec_values, spec.server_version, env) + service = _serve(moved, spec, control, env) if serve else {"launcher": "help_only"} + verify(spec, moved) + check_ops(candidate / "ops") + result = { + "status": "candidate_smoke_passed", + "qualified": False, + "resource_digest": spec.digest, + "archive_sha256": sha256(candidate / "server/server.tar.gz"), + "catalog_sha256": catalog_digest, + "server": service, + "ops": "independent_inspection_passed", + "http_download": "not_executed", + "native_service_manager_lifecycle": "not_executed", + } + print(json.dumps(result, ensure_ascii=False)) + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument( + "--evidence-dir", + type=Path, + help="Durable bounded failure logs; defaults to a sibling candidate-evidence directory", + ) + parser.add_argument( + "--serve", + action="store_true", + help="Start the actual launcher with disposable SQLite data; never register a service", + ) + args = parser.parse_args() + check(args.candidate, serve=args.serve, evidence_dir=args.evidence_dir) + + +if __name__ == "__main__": + main() diff --git a/scripts/managed_resource_build_tools.json b/scripts/managed_resource_build_tools.json new file mode 100644 index 0000000000..890be71b6c --- /dev/null +++ b/scripts/managed_resource_build_tools.json @@ -0,0 +1,93 @@ +{ + "format_version": 1, + "packages": [ + { + "name": "pip", + "version": "26.2.2", + "filename": "pip-26.2.2-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/3f/06/3e4420302363fbb71be3c7f35ce042f54bfca330e3e66b3a2e3c4cca579f/pip-26.2.2-py3-none-any.whl", + "sha256": "fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70", + "size": 1819850 + }, + { + "name": "setuptools", + "version": "80.9.0", + "filename": "setuptools-80.9.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", + "sha256": "062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", + "size": 1201486 + }, + { + "name": "wheel", + "version": "0.48.0", + "filename": "wheel-0.48.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/2e/29/69cfbb602cd91690c55d38ba9fe53e6a7e76a6fa647bf38f19c138d25449/wheel-0.48.0-py3-none-any.whl", + "sha256": "3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab", + "size": 33320 + }, + { + "name": "hatchling", + "version": "1.33.0", + "filename": "hatchling-1.33.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/32/64/4afe5dd77b2939a93e8c9513f8082dbd049ecba9c1484a379a0f6ae29d6b/hatchling-1.33.0-py3-none-any.whl", + "sha256": "415290b84f464d2884303fca8974d38e7b85579069fab02a15db0828fbb1446d", + "size": 78750 + }, + { + "name": "hatch-vcs", + "version": "0.5.0", + "filename": "hatch_vcs-0.5.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/5f/48/1f85cee4b7b4f40b9b814b1febbc661bda6ced9649e410a0b74f6e415dd0/hatch_vcs-0.5.0-py3-none-any.whl", + "sha256": "b49677dbdc597460cc22d01b27ab3696f5e16a21ecf2700fb01bc28e1f2a04a7", + "size": 8507 + }, + { + "name": "setuptools-scm", + "version": "8.3.1", + "filename": "setuptools_scm-8.3.1-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/ab/ac/8f96ba9b4cfe3e4ea201f23f4f97165862395e9331a424ed325ae37024a8/setuptools_scm-8.3.1-py3-none-any.whl", + "sha256": "332ca0d43791b818b841213e76b1971b7711a960761c5bea5fc5cdb5196fbce3", + "size": 43935 + }, + { + "name": "packaging", + "version": "26.3", + "filename": "packaging-26.3-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", + "sha256": "d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", + "size": 129956 + }, + { + "name": "pathspec", + "version": "1.1.1", + "filename": "pathspec-1.1.1-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", + "sha256": "a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", + "size": 57328 + }, + { + "name": "pluggy", + "version": "1.7.0", + "filename": "pluggy-1.7.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", + "sha256": "7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", + "size": 27082 + }, + { + "name": "tomlkit", + "version": "0.15.2", + "filename": "tomlkit-0.15.2-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/da/46/7d84138a60a8bd796825b2d38ebe92d0d16501ca57d501b7c5d677401bbc/tomlkit-0.15.2-py3-none-any.whl", + "sha256": "05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437", + "size": 52225 + }, + { + "name": "trove-classifiers", + "version": "2026.9.21.13", + "filename": "trove_classifiers-2026.9.21.13-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/30/81/0da8afb52a71d0a4f2bd3152357b1a441e393b286374802b9d3addab4ab5/trove_classifiers-2026.9.21.13-py3-none-any.whl", + "sha256": "8b1ff4f9c191b1040b71c37f1e445ab99732911e3cd91de52838453a854d7a17", + "size": 14232 + } + ] +} diff --git a/scripts/managed_resource_intel_build.json b/scripts/managed_resource_intel_build.json new file mode 100644 index 0000000000..accc0c4b48 --- /dev/null +++ b/scripts/managed_resource_intel_build.json @@ -0,0 +1,29 @@ +{ + "format_version": 1, + "target": "macos-x86_64", + "package": "cryptography", + "version": "50.0.0", + "upstream_support": "x86_64 macOS support removed upstream; native candidate build requires separate qualification", + "xcode_version": "16.4", + "xcode_build": "16F6", + "sdk_version": "15.5", + "deployment_target": "10.13", + "artifacts": { + "rust": { + "filename": "rust-1.95.0-x86_64-apple-darwin.tar.xz", + "url": "https://static.rust-lang.org/dist/rust-1.95.0-x86_64-apple-darwin.tar.xz", + "sha256": "998af750fe2c28f0dbf193edc496c9a7407850486968f40ac11a8c8adfb7cfa6" + }, + "openssl": { + "filename": "openssl-4.0.3.tar.gz", + "url": "https://github.com/openssl/openssl/releases/download/openssl-4.0.3/openssl-4.0.3.tar.gz", + "sha256": "325b5c806167c13b40b1ffeadfe0248197c00eccc4cf123ec1e28d2d2fd216d9" + }, + "maturin": { + "filename": "maturin-1.14.1-py3-none-macosx_10_12_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/43/b6/79c881410a3b1c187f7eb3d407aecae646c6a4433d630d72200359015e83/maturin-1.14.1-py3-none-macosx_10_12_x86_64.whl", + "sha256": "f3306078070c1508fd715b9116070cbcaff5959024272a9f1e6f5cb29768b86c", + "size": 10169205 + } + } +} diff --git a/scripts/managed_resource_interpreters.json b/scripts/managed_resource_interpreters.json new file mode 100644 index 0000000000..6c347c3696 --- /dev/null +++ b/scripts/managed_resource_interpreters.json @@ -0,0 +1,37 @@ +{ + "format_version": 1, + "upstream": "https://github.com/astral-sh/python-build-standalone/tree/20261026", + "license": "https://github.com/astral-sh/python-build-standalone/blob/20261026/LICENSE", + "targets": { + "macos-arm64": { + "platform": "macos", + "architecture": "arm64", + "python_version": "3.12.16", + "python_relative": "bin/python3", + "server_python_relative": "bin/python3", + "site_relative": "lib/python3.12/site-packages", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20261026/cpython-3.12.16%2B20261026-aarch64-apple-darwin-install_only.tar.gz", + "sha256": "fbf4afef53c2a6b72bb741bc06d5671a9dfaf87bd84e5aa73ac9b3d0d589f14e" + }, + "macos-x86_64": { + "platform": "macos", + "architecture": "x86_64", + "python_version": "3.12.16", + "python_relative": "bin/python3", + "server_python_relative": "bin/python3", + "site_relative": "lib/python3.12/site-packages", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20261026/cpython-3.12.16%2B20261026-x86_64-apple-darwin-install_only.tar.gz", + "sha256": "a962e47df522889eaa7ff0d6a56203f1079f68aabb65ffdbe8455f362bc769f3" + }, + "windows-x86_64": { + "platform": "windows", + "architecture": "x86_64", + "python_version": "3.12.16", + "python_relative": "python.exe", + "server_python_relative": "pythonw.exe", + "site_relative": "Lib/site-packages", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20261026/cpython-3.12.16%2B20261026-x86_64-pc-windows-msvc-install_only.tar.gz", + "sha256": "590dfcbe3a34c07301e6332e13e2188b98655cd072b08afa45a7998dc4c22bc5" + } + } +} diff --git a/scripts/prepare_managed_resources.py b/scripts/prepare_managed_resources.py new file mode 100644 index 0000000000..405ed7df8f --- /dev/null +++ b/scripts/prepare_managed_resources.py @@ -0,0 +1,566 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Freeze native candidate inputs from an exact Git source and the checked-in locks. + +Network acquisition and the explicitly pinned source builds happen here. +The separate producer consumes only the resulting hashed, offline wheel closure. +""" + +# ruff: noqa: TRY003, S603, S607 - explicit release-producer inputs and fixed build-host tools + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import posixpath +import re +import shutil +import stat +import subprocess +import tarfile +import tempfile +import tomllib +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path, PurePosixPath +from typing import Any +from urllib.parse import unquote, urlsplit + +import build_intel_managed_wheel +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name, parse_wheel_filename +from packaging.version import Version + +from powercontext.ops.resources import MAX_FILES, MAX_UNPACKED_BYTES, ResourceFile + +_SCRIPTS = Path(__file__).resolve().parent +_INTERPRETERS = _SCRIPTS / "managed_resource_interpreters.json" +_BUILD_TOOLS = _SCRIPTS / "managed_resource_build_tools.json" +_SOURCE_REPOSITORY = "https://github.com/oceanbase/powercontext" + + +def json_bytes(value: object) -> bytes: + return (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf-8") + + +def sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def producer_identity() -> dict[str, Any]: + """Bind executable producer sources separately from the packaged Server source.""" + names = ( + "prepare_managed_resources.py", + "produce_managed_resources.py", + "build_ops_bundle.py", + "managed_resource_interpreters.json", + "managed_resource_build_tools.json", + "build_intel_managed_wheel.py", + "managed_resource_intel_build.json", + ) + files = {"scripts/" + name: sha256(_SCRIPTS / name) for name in names} + revision = subprocess.check_output(["git", "-C", str(_SCRIPTS.parent), "rev-parse", "HEAD"], text=True).strip() + matches = True + for relative, digest in files.items(): + committed = subprocess.run( + ["git", "-C", str(_SCRIPTS.parent), "show", f"{revision}:{relative}"], capture_output=True, check=False + ) + matches = matches and committed.returncode == 0 and hashlib.sha256(committed.stdout).hexdigest() == digest + return {"git_commit": revision, "files_sha256": files, "matches_committed_files": matches} + + +def environment() -> dict[str, str]: + return { + key: value + for key, value in os.environ.items() + if not key.upper().startswith(("PYTHON", "PIP_", "UV_", "POWERCONTEXT", "SETUPTOOLS_SCM", "HATCH_")) + } + + +def target_spec(target: str, *, native: bool = True) -> dict[str, Any]: + lock = json.loads(_INTERPRETERS.read_bytes()) + spec = lock["targets"][target] + current = ( + {"Darwin": "macos", "Windows": "windows"}.get(platform.system()), + {"amd64": "x86_64", "x86_64": "x86_64", "arm64": "arm64", "aarch64": "arm64"}.get(platform.machine().lower()), + ) + if native and current != (spec["platform"], spec["architecture"]): + raise ValueError("Candidate production requires the matching native OS and architecture") + return spec + + +def acquire(item: dict[str, Any], destination: Path, cache: Path | None = None) -> None: + """Download fixed HTTPS bytes; incomplete and corrupt files cannot be reused.""" + if destination.exists(): + if sha256(destination) != item["sha256"]: + raise ValueError("Existing input has the wrong digest; inspect it before retrying") + return + if cache is not None: + cached = cache.resolve() / item["sha256"] + acquire(item, cached) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(cached, destination) + return + parsed = urlsplit(item["url"]) + if parsed.scheme != "https" or parsed.username or parsed.password or parsed.fragment: + raise ValueError("Inputs must have fixed HTTPS source URLs") + destination.parent.mkdir(parents=True, exist_ok=True) + partial = destination.with_name(destination.name + ".partial") + if partial.exists(): + raise ValueError("An incomplete input exists; inspect it before retrying") + subprocess.run( + [ + "curl", + "--fail", + "--silent", + "--show-error", + "--location", + "--proto", + "=https", + "--proto-redir", + "=https", + "--retry", + "2", + "--max-time", + "1800", + "--output", + str(partial), + item["url"], + ], + check=True, + env=environment(), + timeout=1900, + ) + if sha256(partial) != item["sha256"] or ("size" in item and partial.stat().st_size != item["size"]): + raise ValueError("Downloaded input does not match its locked digest and size") + partial.rename(destination) + + +def extract_runtime(archive: Path, output: Path) -> list[dict[str, str]]: # noqa: C901 - one archive validation boundary + """Materialize regular files and safe internal file aliases, never archive links. + + ResourceStore deliberately rejects links. Directory aliases, absolute/external + aliases, devices and ambiguous paths are rejected before any output is made. + """ + if output.exists(): + raise ValueError("Runtime output must not exist") + aliases: list[dict[str, str]] = [] + with tarfile.open(archive, "r:gz") as source: + members: dict[str, tarfile.TarInfo] = {} + casefolded: set[str] = set() + total = 0 + for member in source: + name = member.name.rstrip("/") + if name == "python" and member.isdir(): + continue + if not name.startswith("python/"): + raise ValueError("Interpreter archive must contain only the python/ install tree") + relative = name.removeprefix("python/") + ResourceFile(path=relative, size=member.size if member.isfile() else 0, sha256="0" * 64) + if relative.casefold() in casefolded or not ( + member.isfile() or member.isdir() or member.issym() or member.islnk() + ): + raise ValueError("Interpreter archive has duplicate paths or unsupported file types") + members[relative] = member + casefolded.add(relative.casefold()) + total += member.size + if len(members) > MAX_FILES or total > MAX_UNPACKED_BYTES: + raise ValueError("Interpreter archive exceeds resource limits") + + def regular(name: str, seen: frozenset[str] = frozenset()) -> tarfile.TarInfo: + member = members.get(name) + if member is None or name in seen or member.isdir(): + raise ValueError("Runtime aliases must resolve to an internal regular file") + if member.isfile(): + return member + link = member.linkname + if not link or link.startswith("/") or "\\" in link or ":" in link: + raise ValueError("Runtime aliases must be relative and internal") + resolved = posixpath.normpath( + posixpath.join("python", posixpath.dirname(name), link) if member.issym() else link + ) + if not resolved.startswith("python/"): + raise ValueError("Runtime alias escapes the install tree") + return regular(resolved.removeprefix("python/"), seen | {name}) + + files = [(name, regular(name)) for name, member in members.items() if not member.isdir()] + for name, member in files: + for parent in PurePosixPath(name).parents: + if str(parent) in members and not members[str(parent)].isdir(): + raise ValueError("Runtime file shadows a directory") + if not members[name].isfile(): + aliases.append({"path": name, "source": member.name.removeprefix("python/")}) + if sum(member.size for _name, member in files) > MAX_UNPACKED_BYTES: + raise ValueError("Materialized runtime exceeds resource limits") + output.mkdir(parents=True) + for name, member in files: + destination = output / name + destination.parent.mkdir(parents=True, exist_ok=True) + stream = source.extractfile(member) + if stream is None: + raise ValueError("Runtime member has no file content") + with stream, destination.open("xb") as written: + shutil.copyfileobj(stream, written) + executable = bool(member.mode & 0o111) or destination.suffix.lower() == ".exe" + destination.chmod(0o755 if executable else 0o644) + return sorted(aliases, key=lambda item: item["path"]) + + +def inventory(root: Path) -> list[dict[str, Any]]: + files = [] + for path in sorted(root.rglob("*")): + status = path.lstat() + if path.is_symlink() or getattr(status, "st_file_attributes", 0) & 0x400: + raise ValueError("Candidate trees must contain no symlinks or reparse points") + if stat.S_ISDIR(status.st_mode): + continue + if not stat.S_ISREG(status.st_mode) or status.st_nlink != 1: + raise ValueError("Candidate inventory accepts only independent regular files") + files.append({ + "path": path.relative_to(root).as_posix(), + "size": status.st_size, + "sha256": sha256(path), + "executable": bool(status.st_mode & stat.S_IXUSR) or (os.name == "nt" and path.suffix.lower() == ".exe"), + }) + return files + + +def _wheel_identity(path: Path) -> dict[str, Any]: + name, version, _build, _tags = parse_wheel_filename(path.name) + return { + "name": name, + "version": str(version), + "filename": path.name, + "sha256": sha256(path), + "size": path.stat().st_size, + } + + +def _locked_dependencies( # noqa: C901 - exact requirement and platform-artifact selection + export: str, lock: dict[str, Any], markers: dict[str, str], tags: list[str], target: str | None = None +) -> list[dict[str, Any]]: + records = [] + ranks = {tag: index for index, tag in enumerate(tags)} + for line in export.replace("\\\n", " ").splitlines(): + line = line.partition("#")[0].strip() + if not line: + continue + requirement = Requirement(line.split(" --hash=")[0].strip()) + if requirement.marker and not requirement.marker.evaluate(markers): + continue + versions = list(requirement.specifier) + if requirement.url or len(versions) != 1 or versions[0].operator != "==" or "*" in versions[0].version: + raise ValueError("Exported runtime dependencies must have exact registry versions") + name, version = canonicalize_name(requirement.name), versions[0].version + package = next(p for p in lock["package"] if p["name"] == name and p["version"] == version) + candidates = [] + for wheel in package.get("wheels", []): + filename = unquote(PurePosixPath(urlsplit(wheel["url"]).path).name) + _name, _version, _build, wheel_tags = parse_wheel_filename(filename) + rank = min((ranks[str(tag)] for tag in wheel_tags if str(tag) in ranks), default=None) + if rank is not None: + candidates.append((rank, filename, wheel)) + if candidates: + _rank, filename, artifact = min(candidates) + kind = "wheel" + elif (name, version) == ("jieba", "0.42.1"): + artifact = package["sdist"] + filename = "jieba-0.42.1.tar.gz" + kind = "sdist" + elif target == "macos-x86_64" and (name, version) == ("cryptography", "50.0.0"): + artifact = package["sdist"] + filename = "cryptography-50.0.0.tar.gz" + kind = "intel-cryptography-sdist" + else: + raise ValueError( + f"No locked compatible wheel for {name}=={version}; source builds are not implicitly allowed" + ) + digest = artifact["hash"].removeprefix("sha256:") + if f"--hash=sha256:{digest}" not in line: + raise ValueError("Selected artifact is not covered by the exported requirement hashes") + records.append({ + "name": name, + "version": version, + "kind": kind, + "filename": filename, + "url": artifact["url"], + "sha256": digest, + "size": artifact["size"], + }) + return sorted(records, key=lambda item: item["name"]) + + +def prepare(args: argparse.Namespace) -> None: # noqa: C901 - ordered acquisition and source-build stages + spec = target_spec(args.target) + if re.fullmatch(r"[0-9a-f]{40}", args.source_ref) is None: + raise ValueError("Source ref must be an exact 40-character Git commit") + if str(Version(args.server_version)) != args.server_version: + raise ValueError("Server version must be an exact canonical Python package version") + output = args.output.resolve() + if output.exists(): + raise ValueError("Input output must not exist; inspect partial work before retrying") + source_repo = args.source.resolve() + commit = subprocess.check_output( + ["git", "-C", str(source_repo), "rev-parse", args.source_ref + "^{commit}"], text=True + ).strip() + if commit != args.source_ref: + raise ValueError("Source commit does not match the requested source") + output.mkdir(parents=True) + if args.runtime_archive is None: + acquire(spec, output / "python.tar.gz", args.download_cache) + else: + if sha256(args.runtime_archive) != spec["sha256"]: + raise ValueError("Supplied runtime archive does not match the checked-in interpreter pin") + shutil.copyfile(args.runtime_archive, output / "python.tar.gz") + with tempfile.TemporaryDirectory(prefix="pc-resource-prepare-") as temporary: + work = Path(temporary).resolve() + runtime = work / "runtime" + aliases = extract_runtime(output / "python.tar.gz", runtime) + build_lock = json.loads(_BUILD_TOOLS.read_bytes()) + tooling = work / "tooling" + tooling.mkdir() + for item in build_lock["packages"]: + acquire(item, tooling / item["filename"], args.download_cache) + tool_requirements = tooling / "requirements.lock" + tool_requirements.write_text( + "".join(f"{p['name']}=={p['version']} --hash=sha256:{p['sha256']}\n" for p in build_lock["packages"]), + encoding="utf-8", + ) + env = environment() + uv = shutil.which(args.uv) + if uv is None: + raise ValueError("The build host needs uv") + subprocess.run( + [uv, "venv", "--python", str(runtime / spec["python_relative"]), str(work / "build-env")], + check=True, + env=env, + ) + builder = work / "build-env" / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + subprocess.run( + [ + uv, + "--offline", + "pip", + "install", + "--python", + str(builder), + "--no-index", + "--find-links", + str(tooling), + "--require-hashes", + "-r", + str(tool_requirements), + ], + check=True, + env=env, + ) + identity = json.loads( + subprocess.check_output( + [ + str(builder), + "-I", + "-B", + "-c", + "import json; from packaging.markers import default_environment; from packaging.tags import sys_tags; print(json.dumps(dict(markers=default_environment(), tags=[str(t) for t in sys_tags()])))", + ], + env=env, + ) + ) + if identity["markers"]["python_full_version"] != spec["python_version"]: + raise ValueError("Pinned interpreter reports an unexpected Python version") + snapshot = output / "source.tar" + subprocess.run( + ["git", "-C", str(source_repo), "archive", "--format=tar", "--output", str(snapshot), commit], + check=True, + env=env, + ) + source = work / "source" + source.mkdir() + with tarfile.open(snapshot) as archive: + archive.extractall(source, filter="data") + epoch = subprocess.check_output( + ["git", "-C", str(source_repo), "show", "-s", "--format=%ct", commit], text=True + ).strip() + # hatch-vcs invokes get_version without a distribution name, so the + # package-specific setuptools-scm override is not available here. + env.update({"SOURCE_DATE_EPOCH": epoch, "SETUPTOOLS_SCM_PRETEND_VERSION": args.server_version}) + exported = output / "upstream-requirements.lock" + subprocess.run( + [ + uv, + "export", + "--project", + str(source), + "--locked", + "--no-dev", + "--extra", + "ops", + "--extra", + "cli", + "--no-emit-project", + "--no-emit-workspace", + "--no-header", + "--no-annotate", + "--output-file", + str(exported), + ], + check=True, + env=env, + stdout=subprocess.DEVNULL, + ) + locked = tomllib.loads((source / "uv.lock").read_text(encoding="utf-8")) + dependencies = _locked_dependencies( + exported.read_text(encoding="utf-8"), locked, identity["markers"], identity["tags"], args.target + ) + downloads = work / "downloads" + downloads.mkdir() + intel_pins = json.loads(build_intel_managed_wheel.PINS.read_bytes()) + native_artifacts = ( + list(intel_pins["artifacts"].values()) + if any(item["kind"] == "intel-cryptography-sdist" for item in dependencies) + else [] + ) + with ThreadPoolExecutor(max_workers=4) as pool: + list( + pool.map( + lambda item: acquire(item, downloads / item["filename"], args.download_cache), + [*dependencies, *native_artifacts], + ) + ) + wheels = output / "wheels" + wheels.mkdir() + native_builds = [] + for item in dependencies: + if item["kind"] == "wheel": + shutil.copyfile(downloads / item["filename"], wheels / item["filename"]) + elif item["kind"] == "intel-cryptography-sdist": + native_builds.append( + build_intel_managed_wheel.build( + source_archive=downloads / item["filename"], + source_identity=item, + artifacts=downloads, + dependencies=dependencies, + python=builder, + uv=uv, + wheels=wheels, + environment=env, + ) + ) + else: + subprocess.run( + [ + str(builder), + "-I", + "-B", + "-m", + "pip", + "--no-cache-dir", + "wheel", + "--no-deps", + "--no-build-isolation", + "--no-index", + "--wheel-dir", + str(wheels), + str(downloads / item["filename"]), + ], + check=True, + env=env, + cwd=work, + ) + subprocess.run( + [ + uv, + "build", + "--offline", + "--no-build-isolation", + "--python", + str(builder), + "--wheel", + "--no-create-gitignore", + "--out-dir", + str(wheels), + str(source), + ], + check=True, + env=env, + cwd=work, + ) + closure = [_wheel_identity(path) for path in sorted(wheels.glob("*.whl"))] + project = [item for item in closure if item["name"] == "powercontext"] + if len(project) != 1 or project[0]["version"] != args.server_version or len(closure) != len(dependencies) + 1: + raise ValueError("Built wheel closure does not match the requested Server version and dependencies") + requirements = output / "requirements.lock" + requirements.write_text( + "".join( + f"{p['name']}{'[ops,cli]' if p['name'] == 'powercontext' else ''}=={p['version']} --hash=sha256:{p['sha256']}\n" + for p in closure + ), + encoding="utf-8", + newline="\n", + ) + metadata = { + "format_version": 1, + "target": args.target, + "server_version": args.server_version, + "interpreter": spec, + "interpreter_lock_sha256": sha256(_INTERPRETERS), + "source": { + "repository": _SOURCE_REPOSITORY, + "commit": commit, + "archive_sha256": sha256(snapshot), + "uv_lock_sha256": sha256(source / "uv.lock"), + "pyproject_sha256": sha256(source / "pyproject.toml"), + "source_date_epoch": int(epoch), + }, + "build_tools": { + "lock_sha256": sha256(_BUILD_TOOLS), + "packages": build_lock["packages"], + "uv_version": subprocess.check_output([uv, "--version"], text=True).strip(), + }, + "upstream_artifacts": dependencies, + "wheels": closure, + "requirements_sha256": sha256(requirements), + "upstream_requirements_sha256": sha256(exported), + "normalized_runtime_aliases": aliases, + "preparer": producer_identity(), + "intel_build_pins_sha256": sha256(build_intel_managed_wheel.PINS), + "native_builds": native_builds, + } + (output / "inputs.json").write_bytes(json_bytes(metadata)) + print(f"Frozen native candidate inputs: {output}; inputs_sha256={sha256(output / 'inputs.json')}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--target", choices=json.loads(_INTERPRETERS.read_bytes())["targets"], required=True) + parser.add_argument("--source", type=Path, default=_SCRIPTS.parent) + parser.add_argument("--source-ref", required=True, help="Exact source Git commit, never a moving ref") + parser.add_argument("--server-version", required=True, help="Exact package version to embed and verify") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--runtime-archive", type=Path, help="Optional already downloaded archive; the fixed digest is still mandatory" + ) + parser.add_argument("--uv", default="uv") + parser.add_argument( + "--download-cache", type=Path, help="Optional task-local digest-addressed cache; every cache hit is verified" + ) + prepare(parser.parse_args()) + + +if __name__ == "__main__": + main() diff --git a/scripts/produce_managed_resources.py b/scripts/produce_managed_resources.py new file mode 100644 index 0000000000..d58021a5cc --- /dev/null +++ b/scripts/produce_managed_resources.py @@ -0,0 +1,483 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Produce unqualified managed Server and independent Ops candidates offline. + +This does not publish, sign, notarize or qualify a release. Hashes describe the +unsigned candidate bytes; qualified catalogs must be frozen after actual signing. +""" + +# ruff: noqa: TRY003, S603 - explicit release-producer inputs and actionable failures + +from __future__ import annotations + +import argparse +import csv +import email.parser +import gzip +import hashlib +import json +import os +import shutil +import subprocess +import tarfile +import tempfile +import zipfile +from pathlib import Path, PureWindowsPath +from typing import Any +from urllib.parse import urlsplit + +import build_intel_managed_wheel +import build_ops_bundle +from packaging.utils import parse_wheel_filename +from prepare_managed_resources import ( + _BUILD_TOOLS, + _INTERPRETERS, + environment, + extract_runtime, + inventory, + json_bytes, + producer_identity, + sha256, + target_spec, +) + +from powercontext.ops.installer import ManagedLayout +from powercontext.ops.protocol import PROTOCOL_VERSION +from powercontext.ops.resources import ResourceFile, ResourceSpec, validate_destination + +_PROBE = """ +import importlib.metadata, json, platform, sys +from pathlib import Path +root = Path(sys.argv[1]).resolve() +if not sys.flags.isolated or not sys.dont_write_bytecode: + raise RuntimeError("The managed interpreter must use -I -B") +import fastapi, fastmcp, sqlalchemy, sqlite_vec +import powercontext.server.cli, powercontext.service.launcher, powercontext.cli.app +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.backends.openssl.backend import backend +key = AESGCM.generate_key(bit_length=128) +aead = AESGCM(key) +ciphertext = aead.encrypt(bytes(12), b"candidate integrity check", b"candidate") +if aead.decrypt(bytes(12), ciphertext, b"candidate") != b"candidate integrity check": + raise RuntimeError("Bundled cryptography AES-GCM failed") +private = Ed25519PrivateKey.generate() +private.public_key().verify(private.sign(b"candidate"), b"candidate") +outside = [str(Path(m.__file__).resolve()) for m in list(sys.modules.values()) + if getattr(m, "__file__", None) and not m.__file__.startswith("<") + and not Path(m.__file__).resolve().is_relative_to(root)] +if outside or any(not Path(p).resolve().is_relative_to(root) for p in sys.path): + raise RuntimeError("Managed Server imports or paths escaped its artifact: " + repr(outside)) +print(json.dumps(dict(python_version=platform.python_version(), + server_version=importlib.metadata.version("powercontext"), + platform=sys.platform, architecture=platform.machine(), outside=outside, + cryptography_version=importlib.metadata.version("cryptography"), + openssl_version=backend.openssl_version_text(), crypto_self_check="passed"))) +""" + + +def verify_inputs(root: Path, expected_sha256: str) -> dict[str, Any]: + if sha256(root / "inputs.json") != expected_sha256: + raise ValueError("Prepared inputs do not match the caller-pinned digest") + metadata = json.loads((root / "inputs.json").read_bytes()) + spec = target_spec(metadata["target"]) + if ( + metadata["format_version"] != 1 + or metadata["interpreter"] != spec + or metadata["interpreter_lock_sha256"] != sha256(_INTERPRETERS) + or metadata["build_tools"]["lock_sha256"] != sha256(_BUILD_TOOLS) + or metadata["intel_build_pins_sha256"] != sha256(build_intel_managed_wheel.PINS) + or sha256(root / "python.tar.gz") != spec["sha256"] + or sha256(root / "source.tar") != metadata["source"]["archive_sha256"] + or sha256(root / "requirements.lock") != metadata["requirements_sha256"] + or sha256(root / "upstream-requirements.lock") != metadata["upstream_requirements_sha256"] + ): + raise ValueError("Prepared source, interpreter or requirement identity is inconsistent") + found = {path.name for path in (root / "wheels").iterdir()} + expected = {item["filename"] for item in metadata["wheels"]} + if found != expected or len(expected) != len(metadata["wheels"]): + raise ValueError("Wheelhouse must contain exactly the frozen wheel closure") + for item in metadata["wheels"]: + path = root / "wheels" / item["filename"] + if ( + path.name != item["filename"] + or path.is_symlink() + or sha256(path) != item["sha256"] + or path.stat().st_size != item["size"] + ): + raise ValueError("A frozen wheel has changed") + name, version, _build, _tags = parse_wheel_filename(path.name) + if (name, str(version)) != (item["name"], item["version"]): + raise ValueError("Frozen wheel identity does not match its filename") + project = [item for item in metadata["wheels"] if item["name"] == "powercontext"] + if len(project) != 1 or project[0]["version"] != metadata["server_version"]: + raise ValueError("Frozen wheel closure does not match the requested Server version") + requirements = "".join( + f"{p['name']}{'[ops,cli]' if p['name'] == 'powercontext' else ''}=={p['version']} --hash=sha256:{p['sha256']}\n" + for p in metadata["wheels"] + ).encode("utf-8") + if (root / "requirements.lock").read_bytes() != requirements: + raise ValueError("Requirements must describe exactly the frozen wheel closure and Ops/CLI extras") + return metadata + + +def deterministic_archive(root: Path, output: Path, files: list[dict[str, Any]]) -> None: + """Stable ordering, permissions, owner, timestamp, gzip header and file bytes.""" + with ( + output.open("xb") as raw, + gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed, + tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive, + ): + for file in files: + member = tarfile.TarInfo(file["path"]) + member.size = file["size"] + member.mode = 0o755 if file["executable"] else 0o644 + with (root / file["path"]).open("rb") as source: + archive.addfile(member, source) + + +def _remove_generated_wrappers(packages: Path) -> None: + """The managed profile launches modules, not installer-generated console wrappers. + + uv's generated shebangs/Windows launchers bind the build path. Remove only + those wrapper directories and repair their RECORD rows before freezing bytes. + uv cache records contain mutable build-input timestamps, so they also + stay out of the sealed runtime. Importable modules and wheel data remain. + """ + for name in ("bin", "Scripts"): + scripts = packages / name + if scripts.exists(): + shutil.rmtree(scripts) + for cache in packages.glob("*.dist-info/uv_cache.json"): + cache.unlink() + for record in packages.glob("*.dist-info/RECORD"): + with record.open(newline="", encoding="utf-8") as stream: + rows = [row for row in csv.reader(stream) if (packages / row[0]).is_file()] + with record.open("w", newline="", encoding="utf-8") as stream: + csv.writer(stream, lineterminator="\n").writerows(sorted(rows)) + + +def _licenses(inputs: Path, runtime: Path, metadata: dict[str, Any]) -> dict[str, Any]: + packages = [] + for item in metadata["wheels"]: + with zipfile.ZipFile(inputs / "wheels" / item["filename"]) as wheel: + path = next(name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")) + message = email.parser.Parser().parsestr(wheel.read(path).decode("utf-8")) + notices = [ + name + for name in wheel.namelist() + if any(part.lower().startswith(("license", "copying", "notice")) for part in Path(name).parts) + ] + packages.append({ + "name": item["name"], + "version": item["version"], + "wheel_sha256": item["sha256"], + "license_expression": message.get("License-Expression"), + "license_text": message.get("License"), + "classifiers": message.get_all("Classifier", []), + "notices": [ + {"wheel_path": name, "sha256": hashlib.sha256(wheel.read(name)).hexdigest()} + for name in sorted(notices) + if not name.endswith("/") + ], + }) + runtime_notices = [ + file + for file in inventory(runtime) + if any(part.lower().startswith(("license", "copying", "notice")) for part in Path(file["path"]).parts) + ] + if not runtime_notices: + raise ValueError("The redistributable interpreter must retain its in-tree license notices") + lock = json.loads(_INTERPRETERS.read_bytes()) + return { + "format_version": 1, + "runtime_license": lock["license"], + "runtime_notices": runtime_notices, + "packages": packages, + } + + +def _server_probe(root: Path, spec: dict[str, Any], version: str, environment_values: dict[str, str]) -> None: + python = root / spec["python_relative"] + result = subprocess.run( + [str(python), "-I", "-B", "-c", _PROBE, str(root)], + cwd=root, + env=environment_values, + check=True, + capture_output=True, + text=True, + timeout=180, + ) + identity = json.loads(result.stdout) + if identity["python_version"] != spec["python_version"] or identity["server_version"] != version: + raise ValueError("Installed Server or interpreter version does not match the frozen input") + architecture = {"amd64": "x86_64", "x86_64": "x86_64", "arm64": "arm64", "aarch64": "arm64"}.get( + identity["architecture"].lower() + ) + if ( + identity["platform"] != {"macos": "darwin", "windows": "win32"}[spec["platform"]] + or architecture != spec["architecture"] + ): + raise ValueError("Installed runtime platform does not match its resource identity") + for relative in dict.fromkeys((spec["python_relative"], spec["server_python_relative"])): + subprocess.run( + [str(root / relative), "-I", "-B", "-m", "powercontext.service.launcher", "--help"], + cwd=root, + env=environment_values, + check=True, + capture_output=True, + timeout=90, + ) + + +def measure_windows_paths(root: PureWindowsPath, files: list[dict[str, Any]]) -> dict[str, Any]: + """Measure ordinary Win32 file/directory limits, including UTF-16 units.""" + units = lambda value: len(str(value).encode("utf-16-le")) // 2 + if not root.is_absolute() or str(root).startswith("\\\\?\\"): + raise ValueError("Windows candidate paths must use an ordinary absolute root") + directories = {root} + paths = [root / file["path"] for file in files] + for path in paths: + directories.update(path.parents) + if units(path) > 259 or any(units(part) > 255 for part in path.parts): + raise ValueError("Candidate does not fit the actual Windows default/staging path budget") + if any(units(path) > 247 for path in directories): + raise ValueError("Candidate does not fit the actual Windows directory creation budget") + return {"root": str(root), "longest_file_utf16_units": max(map(units, paths))} + + +def _windows_path_budget( + files: list[dict[str, Any]], resource_id: str, ops_files: list[dict[str, Any]] +) -> dict[str, Any]: + if os.name != "nt": + return {"status": "not_executed", "reason": "requires native Windows paths"} + layout = ManagedLayout.current_user() + # Reuse the same public path validator as actual acquisition and preflight. + # The spec is constructed by the caller; this helper reports the measured roots. + roots = { + "server_final": layout.programs / "server" / resource_id, + "server_staging": layout.programs / "staging" / ("resource-" + "0" * 32), + "ops_final": Path(os.environ["LOCALAPPDATA"]) / "PowerContext/Ops/bundle", + } + lengths = {} + for name, root in roots.items(): + chosen = ops_files if name == "ops_final" else files + lengths[name] = measure_windows_paths(PureWindowsPath(str(root)), chosen) + return {"status": "passed", "measurements": lengths} + + +def produce(args: argparse.Namespace) -> None: + inputs = args.inputs.resolve() + output = args.output.resolve() + if output.exists(): + raise ValueError("Output must not exist; candidate production never overwrites an artifact") + if output.is_relative_to(inputs) or inputs.is_relative_to(output): + raise ValueError("Inputs and output must not contain one another") + metadata = verify_inputs(inputs, args.inputs_sha256) + spec = metadata["interpreter"] + parsed = urlsplit(args.server_url) + origin = f"https://{parsed.netloc}" + if parsed.scheme != "https" or parsed.username or parsed.password or parsed.fragment: + raise ValueError("A fixed candidate HTTPS archive URL is required; no release URL is inferred") + uv = shutil.which(args.uv) + if uv is None: + raise ValueError("The build host needs uv for offline wheel installation") + output.mkdir(parents=True) + env = environment() + with tempfile.TemporaryDirectory(prefix="pc-candidate-") as temporary: + work = Path(temporary).resolve() + runtime = work / "python" + aliases = extract_runtime(inputs / "python.tar.gz", runtime) + if aliases != metadata["normalized_runtime_aliases"]: + raise ValueError("Runtime alias normalization differs from the prepared input") + licenses = _licenses(inputs, runtime, metadata) + server = work / "server" + shutil.copytree(runtime, server) + site = server / spec["site_relative"] + subprocess.run( + [ + uv, + "--offline", + "pip", + "install", + "--python", + str(server / spec["python_relative"]), + "--target", + str(site), + "--link-mode", + "copy", + "--no-index", + "--find-links", + str(inputs / "wheels"), + "--only-binary", + ":all:", + "--require-hashes", + "-r", + str(inputs / "requirements.lock"), + ], + cwd=work, + env=env, + check=True, + timeout=600, + ) + _remove_generated_wrappers(site) + (server / "resource-licenses.json").write_bytes(json_bytes(licenses)) + # Move once before execution to test relocation away from installer paths. + relocated = work / "relocated server" + server.rename(relocated) + poison = work / "poison" + poison.mkdir() + (poison / "powercontext.py").write_text("raise RuntimeError('developer import leaked')\n", encoding="utf-8") + isolated = {**env, "PATH": "", "PYTHONPATH": str(poison), "PYTHONHOME": str(poison / "missing")} + _server_probe(relocated, spec, metadata["server_version"], isolated) + ops = output / "ops" + build_ops_bundle.build( + argparse.Namespace( + runtime=runtime, + python_relative=spec["python_relative"], + wheelhouse=inputs / "wheels", + requirements=inputs / "requirements.lock", + output=ops, + uv=uv, + ) + ) + _remove_generated_wrappers(ops / "packages") + (ops / "resource-licenses.json").write_bytes(json_bytes(licenses)) + manifest = json.loads((ops / "bundle.json").read_bytes()) + (ops / "bundle.json").unlink() + ops_content = inventory(ops) + manifest.update( + installed_bytes=sum(file["size"] for file in ops_content), + files_sha256={file["path"]: file["sha256"] for file in ops_content}, + ) + (ops / "bundle.json").write_bytes(json_bytes(manifest)) + ops_files = inventory(ops) + deterministic_archive(ops, output / "ops.tar.gz", ops_files) + ops_archive_digest = sha256(output / "ops.tar.gz") + files = inventory(relocated) + server_output = output / "server" + server_output.mkdir() + archive = server_output / "server.tar.gz" + deterministic_archive(relocated, archive, files) + archive_digest = sha256(archive) + resource_id = "pc-" + archive_digest[:24] + binding = { + "inputs_sha256": args.inputs_sha256, + "server_archive_sha256": archive_digest, + "ops_bundle_manifest_sha256": sha256(ops / "bundle.json"), + "ops_archive_sha256": ops_archive_digest, + "server_wheel_sha256": next( + item["sha256"] for item in metadata["wheels"] if item["name"] == "powercontext" + ), + "source_commit": metadata["source"]["commit"], + "server_version": metadata["server_version"], + "python_source_sha256": spec["sha256"], + "producer": producer_identity(), + } + candidate_digest = hashlib.sha256(json_bytes(binding)).hexdigest() + license_digest = hashlib.sha256(json_bytes(licenses)).hexdigest() + resource = ResourceSpec( + resource_id=resource_id, + server_version=metadata["server_version"], + python_version=spec["python_version"], + platform=spec["platform"], + architecture=spec["architecture"], + archive_format="tar.gz", + url=args.server_url, + allowed_origins=(origin,), + archive_sha256=archive_digest, + archive_size=archive.stat().st_size, + unpacked_size=sum(file["size"] for file in files), + python_relative=spec["server_python_relative"], + ops_protocol_min=PROTOCOL_VERSION, + ops_protocol_max=PROTOCOL_VERSION, + source_refs=( + f"git+{metadata['source']['repository']}@{metadata['source']['commit']}", + spec["url"], + "inputs-sha256:" + args.inputs_sha256, + ), + license_refs=("resource-licenses.json#sha256=" + license_digest,), + qualification_ref="unqualified-candidate:sha256:" + candidate_digest, + files=tuple(ResourceFile.model_validate(file) for file in files), + ) + if os.name == "nt": + layout = ManagedLayout.current_user() + validate_destination(resource, layout.programs / "server" / resource_id) + validate_destination(resource, layout.programs / "staging" / ("resource-" + "0" * 32)) + path_budget = _windows_path_budget(files, resource_id, ops_files) + catalog = server_output / "catalog.json" + catalog.write_bytes(json_bytes({"format_version": 1, "resources": [resource.model_dump(mode="json")]})) + catalog_digest = sha256(catalog) + (server_output / "catalog.sha256").write_text(catalog_digest + "\n", encoding="utf-8") + (server_output / "resource.json").write_bytes( + json_bytes({ + "format_version": 1, + "status": "unqualified_candidate", + "resource": resource.model_dump(mode="json"), + "inputs": metadata, + "inputs_sha256": args.inputs_sha256, + }) + ) + (output / "ops-resource.json").write_bytes( + json_bytes({ + "format_version": 1, + "status": "unqualified_candidate", + "files": ops_files, + "inputs_sha256": args.inputs_sha256, + "archive_sha256": ops_archive_digest, + "archive_size": (output / "ops.tar.gz").stat().st_size, + }) + ) + (output / "ops-provision.json").write_bytes( + json_bytes({ + "format_version": 1, + "bundle_manifest_sha256": sha256(ops / "bundle.json"), + "binding_mode": "managed_catalog", + "catalog_sha256": catalog_digest, + }) + ) + (output / "candidate.json").write_bytes( + json_bytes({ + "format_version": 1, + "status": "unqualified_candidate", + "qualification_ref": resource.qualification_ref, + "binding": binding, + "catalog_sha256": catalog_digest, + "ops_archive_sha256": ops_archive_digest, + "windows_path_budget": path_budget, + "signing": "not_performed", + "notarization": "not_performed", + "clean_machine_qualification": "not_performed", + }) + ) + print(f"Produced unqualified {metadata['target']} candidate: {output}; catalog_sha256={catalog_digest}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--inputs", type=Path, required=True) + parser.add_argument( + "--inputs-sha256", required=True, help="Expected exact inputs.json SHA-256 from trusted preparation" + ) + parser.add_argument( + "--server-url", required=True, help="Fixed candidate URL supplied by the operator; never inferred" + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--uv", default="uv") + produce(parser.parse_args()) + + +if __name__ == "__main__": + main() diff --git a/tests/test_managed_resource_producer.py b/tests/test_managed_resource_producer.py new file mode 100644 index 0000000000..9658a79044 --- /dev/null +++ b/tests/test_managed_resource_producer.py @@ -0,0 +1,505 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Producer regressions; synthetic archives are not native platform qualification.""" + +from __future__ import annotations + +import hashlib +import io +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import zipfile +from pathlib import Path, PureWindowsPath +from typing import Any + +import httpx +import pytest +from build_intel_managed_wheel import _check_linkage, native_build_environment, verify_backend +from ci_managed_resource_smoke import MAX_DIAGNOSTIC_BYTES, preserve_failure +from prepare_managed_resources import ( + _locked_dependencies, + acquire, + extract_runtime, + inventory, + json_bytes, + sha256, + target_spec, +) +from produce_managed_resources import deterministic_archive, measure_windows_paths + +from powercontext.ops import resources +from powercontext.ops.resources import ResourceCatalog, ResourceError, ResourceSpec, ResourceStore, verify +from powercontext.service.environment import protect_directory, write_protected_file + +_ROOT = Path(__file__).resolve().parents[1] + + +def _runtime_archive(path: Path, *, alias: str = "python.exe", directory_alias: bool = False) -> None: + with tarfile.open(path, "w:gz") as archive: + for name, content in { + "python/python.exe": b"console interpreter fixture", + "python/LICENSE.txt": b"fixture license", + }.items(): + file = tarfile.TarInfo(name) + file.size = len(content) + file.mode = 0o755 if name.endswith(".exe") else 0o644 + archive.addfile(file, io.BytesIO(content)) + if directory_alias: + directory = tarfile.TarInfo("python/lib") + directory.type = tarfile.DIRTYPE + archive.addfile(directory) + file = tarfile.TarInfo("python/pythonw.exe") + file.type = tarfile.SYMTYPE + file.linkname = alias + archive.addfile(file) + + +def _spec(archive: Path, files: list[dict[str, Any]]) -> ResourceSpec: + return ResourceSpec.model_validate({ + "resource_id": "synthetic-candidate", + "server_version": "1.2.3", + "python_version": "3.12.16", + "platform": "windows", + "architecture": "x86_64", + "archive_format": "tar.gz", + "url": "https://candidate.invalid/server.tar.gz", + "allowed_origins": ["https://candidate.invalid"], + "archive_sha256": sha256(archive), + "archive_size": archive.stat().st_size, + "unpacked_size": sum(file["size"] for file in files), + "python_relative": "pythonw.exe", + "ops_protocol_min": 1, + "ops_protocol_max": 1, + "source_refs": ["fixture-source"], + "license_refs": ["LICENSE.txt"], + "qualification_ref": "unqualified-candidate:fixture", + "files": files, + }) + + +def test_internal_file_alias_is_materialized_and_consumed_with_pinned_catalog( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + tmp_path = tmp_path.resolve() + source = tmp_path / "python.tar.gz" + _runtime_archive(source) + tree = tmp_path / "python" + aliases = extract_runtime(source, tree) + assert aliases == [{"path": "pythonw.exe", "source": "python.exe"}] + assert not (tree / "pythonw.exe").is_symlink() + assert (tree / "pythonw.exe").read_bytes() == (tree / "python.exe").read_bytes() + archive = tmp_path / "server.tar.gz" + files = inventory(tree) + deterministic_archive(tree, archive, files) + spec = _spec(archive, files) + control = protect_directory(tmp_path / "control", create=True) + content = json_bytes({"format_version": 1, "resources": [spec.model_dump(mode="json")]}) + write_protected_file(control / "catalog.json", content) + catalog = ResourceCatalog.read(control / "catalog.json", hashlib.sha256(content).hexdigest()) + monkeypatch.setattr(resources, "_current_platform", lambda: ("windows", "x86_64")) + client = httpx.Client + monkeypatch.setattr( + resources.httpx, + "Client", + lambda **kwargs: client( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, stream=httpx.ByteStream(archive.read_bytes())) + ), + **kwargs, + ), + ) + prepared = ResourceStore(tmp_path / "cache").prepare(catalog.select_current(), tmp_path / "staging") + verify(spec, prepared.directory) + assert inventory(prepared.directory) == files + (prepared.directory / "python.exe").write_bytes(b"corruption") + with pytest.raises(ResourceError, match="resource_corrupt"): + verify(spec, prepared.directory) + with pytest.raises(ResourceError, match="resource_catalog_invalid"): + ResourceCatalog.read(control / "catalog.json", "0" * 64) + monkeypatch.setattr(resources, "_current_platform", lambda: ("macos", "arm64")) + with pytest.raises(ResourceError, match="resource_unavailable"): + catalog.select_current() + + +@pytest.mark.parametrize( + "alias,directory", [("../outside", False), ("/absolute/python.exe", False), ("lib", True), ("pythonw.exe", False)] +) +def test_unsafe_or_directory_runtime_alias_rejected_before_extraction( + tmp_path: Path, alias: str, directory: bool +) -> None: + source = tmp_path / "python.tar.gz" + _runtime_archive(source, alias=alias, directory_alias=directory) + with pytest.raises(ValueError, match="alias"): + extract_runtime(source, tmp_path / "output") + assert not (tmp_path / "output").exists() + + +def test_archive_is_deterministic_across_input_roots_and_timestamps(tmp_path: Path) -> None: + archives = [] + for index in range(2): + tree = tmp_path / f"root-{index}" + tree.mkdir() + file = tree / "python.exe" + file.write_bytes(b"same candidate bytes") + file.chmod(0o755) + os.utime(file, (index + 1, index + 1)) + archive = tmp_path / f"artifact-{index}.tar.gz" + deterministic_archive(tree, archive, inventory(tree)) + archives.append(archive.read_bytes()) + assert archives[0] == archives[1] + with pytest.raises(FileExistsError): + deterministic_archive(tree, archive, inventory(tree)) + + +def test_corrupt_cached_interpreter_is_rejected_without_network(tmp_path: Path) -> None: + artifact = tmp_path / "python.tar.gz" + artifact.write_bytes(b"bad existing input") + with pytest.raises(ValueError, match="wrong digest"): + acquire({"url": "https://candidate.invalid/python.tar.gz", "sha256": "0" * 64}, artifact) + assert artifact.read_bytes() == b"bad existing input" + + +def test_windows_layout_requires_console_and_windowless_interpreters(tmp_path: Path) -> None: + source = tmp_path / "python.tar.gz" + _runtime_archive(source) + tree = tmp_path / "python" + extract_runtime(source, tree) + files = inventory(tree) + archive = tmp_path / "server.tar.gz" + deterministic_archive(tree, archive, files) + spec = _spec(archive, files) + payload = spec.model_dump(mode="json") + payload["files"] = [item for item in payload["files"] if item["path"] != "python.exe"] + payload["unpacked_size"] = sum(item["size"] for item in payload["files"]) + with pytest.raises(ValueError, match="resource_invalid"): + ResourceSpec.model_validate(payload) + + +def test_windows_path_budget_covers_real_default_staging_and_utf16() -> None: + root = PureWindowsPath( + "C:/Users/PowerContext User/AppData/Local/Programs/PowerContext/managed/staging/resource-" + "0" * 32 + ) + files = [{"path": "Lib/site-packages/anthropic/types/beta/" + "a" * 75 + ".py"}] + measured = measure_windows_paths(root, files) + assert measured["longest_file_utf16_units"] <= 259 + with pytest.raises(ValueError, match="budget"): + measure_windows_paths(root, [{"path": "Lib/" + "📦" * 80 + ".py"}]) + with pytest.raises(ValueError, match="directory"): + measure_windows_paths(PureWindowsPath("C:/" + "a" * 245), [{"path": "x"}]) + + +def test_foreign_target_is_not_a_cross_compilation_request() -> None: + for target in ("macos-arm64", "macos-x86_64", "windows-x86_64"): + target_spec(target, native=False) + matching = {"darwin": "macos", "win32": "windows"}.get(sys.platform) + foreign = "windows-x86_64" if matching != "windows" else "macos-arm64" + with pytest.raises(ValueError, match="matching native"): + target_spec(foreign) + + +def test_intel_crypto_source_exception_is_version_and_target_specific() -> None: + digest = "a" * 64 + lock = { + "package": [ + { + "name": "cryptography", + "version": "50.0.0", + "sdist": { + "url": "https://candidate.invalid/cryptography-50.0.0.tar.gz", + "hash": "sha256:" + digest, + "size": 10, + }, + } + ] + } + export = "cryptography==50.0.0 --hash=sha256:" + digest + selected = _locked_dependencies(export, lock, {}, ["cp312-cp312-macosx_10_13_x86_64"], "macos-x86_64") + assert selected[0]["kind"] == "intel-cryptography-sdist" + assert selected[0]["sha256"] == digest + with pytest.raises(ValueError, match="source builds are not implicitly allowed"): + _locked_dependencies(export, lock, {}, ["cp312-cp312-win_amd64"], "windows-x86_64") + with pytest.raises(ValueError, match="exported requirement hashes"): + _locked_dependencies( + export.replace(digest, "b" * 64), lock, {}, ["cp312-cp312-macosx_10_13_x86_64"], "macos-x86_64" + ) + + +def test_native_build_cannot_be_retargeted_by_ambient_library_or_backend_options(tmp_path: Path) -> None: + ambient = { + "PATH": str(tmp_path / "untrusted"), + "X86_64_APPLE_DARWIN_OPENSSL_DIR": str(tmp_path / "untrusted-openssl"), + "X86_64_APPLE_DARWIN_OPENSSL_LIB_DIR": str(tmp_path / "untrusted-lib"), + "PYO3_CONFIG_FILE": str(tmp_path / "untrusted-python.cfg"), + "MATURIN_PEP517_ARGS": "--features untrusted", + "CC_x86_64_apple_darwin": str(tmp_path / "untrusted-compiler"), + "CPATH": str(tmp_path / "untrusted-headers"), + "PKG_CONFIG_PATH": str(tmp_path / "untrusted-config"), + "CONDA_PREFIX": str(tmp_path / "untrusted-conda"), + "VIRTUAL_ENV": str(tmp_path / "untrusted-venv"), + "MAKEFLAGS": "-f untrusted.make", + "HTTPS_PROXY": "https://proxy.invalid", + } + selected = native_build_environment( + ambient, + python=tmp_path / "backend/bin/python", + rust=tmp_path / "rust", + workspace=tmp_path, + compiler="/fixed/clang", + sdk_path="/fixed/SDK", + deployment="10.13", + ) + assert not any(name in selected for name in ambient if name not in {"PATH", "HTTPS_PROXY"}) + assert selected["CC"] == "/fixed/clang" + assert selected["SDKROOT"] == "/fixed/SDK" + assert selected["HTTPS_PROXY"] == "https://proxy.invalid" + assert str(tmp_path / "untrusted") not in selected["PATH"] + + +@pytest.mark.skipif(os.name == "nt", reason="Intel backend uses POSIX executable lookup") +def test_bare_backend_command_resolves_the_hash_pinned_wheel_binary(tmp_path: Path) -> None: + python = tmp_path / "backend/bin/python" + python.parent.mkdir(parents=True) + executable = python.parent / "maturin" + content = f"#!{sys.executable}\nprint('fixed backend')\n".encode() + executable.write_bytes(content) + executable.chmod(0o755) + archive = tmp_path / "maturin.whl" + with zipfile.ZipFile(archive, "w") as wheel: + wheel.writestr("maturin-1.14.1.data/scripts/maturin", content) + selected = native_build_environment( + {"PATH": str(tmp_path / "ambient")}, + python=python, + rust=tmp_path / "rust", + workspace=tmp_path, + compiler="/fixed/clang", + sdk_path="/fixed/SDK", + deployment="10.13", + ) + assert verify_backend(archive, python, selected) == hashlib.sha256(content).hexdigest() + result = subprocess.run( + ["maturin", "pep517"], # noqa: S607 - exercise the backend's bare-command executable lookup + env=selected, + check=True, + capture_output=True, + text=True, + ) + assert result.stdout.strip() == "fixed backend" + executable.write_bytes(b"changed backend") + with pytest.raises(ValueError, match="pinned maturin executable"): + verify_backend(archive, python, selected) + + +def test_smoke_failure_keeps_bounded_logs_after_temporary_cleanup(tmp_path: Path) -> None: + evidence = tmp_path / "evidence" + with ( + pytest.raises(RuntimeError, match="bounded diagnostics") as failed, + tempfile.TemporaryDirectory() as temporary, + preserve_failure(evidence, Path(temporary)), + ): + workspace = Path(temporary) + (workspace / "control").mkdir() + (workspace / "control/stderr.log").write_bytes(b"x" * (MAX_DIAGNOSTIC_BYTES + 20) + b"useful tail") + message = "raw error content must not appear inline" + raise ValueError(message) + assert not workspace.exists() + saved = next(evidence.iterdir()) + assert (saved / "stderr.log").stat().st_size == MAX_DIAGNOSTIC_BYTES + assert (saved / "stderr.log").read_bytes().endswith(b"useful tail") + assert json.loads((saved / "failure.json").read_bytes())["exception_class"] == "ValueError" + assert "raw error content" not in str(failed.value) + + +@pytest.mark.skipif(sys.platform != "darwin", reason="requires real Mach-O compiler and load-command inspection") +@pytest.mark.parametrize("architecture", ["arm64", "x86_64"]) +@pytest.mark.parametrize("external", [None, "dependency", "rpath"]) +def test_native_linkage_distinguishes_install_name_from_loaded_dependencies( + tmp_path: Path, architecture: str, external: str | None +) -> None: + tree = tmp_path / "wheel" + tree.mkdir() + source = tmp_path / "extension.c" + source.write_text("int resource_probe(void) { return 1; }\n", encoding="utf-8") + sdk = ( + os.environ.get("SDKROOT") + or subprocess.check_output(["/usr/bin/xcrun", "--sdk", "macosx", "--show-sdk-path"], text=True).strip() + ) + compiler = ["/usr/bin/clang", "-dynamiclib", "-arch", architecture, "-isysroot", sdk] + arguments = [] + if external == "dependency": + dependency = tmp_path / "libexternal.dylib" + subprocess.run( + [ + *compiler, + str(source), + "-Wl,-install_name,/opt/producer-fixture/libexternal.dylib", + "-o", + str(dependency), + ], + check=True, + capture_output=True, + ) + source.write_text("extern int resource_probe(void); int extension(void) { return resource_probe(); }\n") + arguments.append(str(dependency)) + elif external == "rpath": + arguments.append("-Wl,-rpath,/opt/producer-fixture/lib") + subprocess.run( + [*compiler, str(source), *arguments, "-Wl,-install_name,@rpath/extension.so", "-o", str(tree / "extension.so")], + check=True, + capture_output=True, + ) + if external is not None: + with pytest.raises(ValueError, match="non-system libraries or runtime search paths"): + _check_linkage(tree, dict(os.environ)) + else: + (linked,) = _check_linkage(tree, dict(os.environ)) + assert linked["install_names"] == ["@rpath/extension.so"] + assert linked["dependencies"] and all(path.startswith("/usr/lib/") for path in linked["dependencies"]) + assert not linked["rpaths"] + + +@pytest.fixture(scope="module") +def native_candidates(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path, Path, str]: + selected = os.environ.get("POWERCONTEXT_RESOURCE_TEST_INPUTS") + if not selected: + pytest.skip("requires explicitly prepared native pinned inputs; synthetic tests do not qualify a platform") + inputs = Path(selected).resolve() + expected = os.environ["POWERCONTEXT_RESOURCE_TEST_INPUTS_SHA256"] + root = tmp_path_factory.mktemp("native-resource-candidates").resolve() + output = os.environ.get("POWERCONTEXT_RESOURCE_TEST_OUTPUT") + outputs = (Path(output).resolve() if output else root / "first", root / "second") + relocated_inputs = root / "relocated-inputs" + shutil.copytree(inputs, relocated_inputs, copy_function=shutil.copyfile) + for output, prepared_inputs in zip(outputs, (inputs, relocated_inputs), strict=True): + subprocess.run( + [ + sys.executable, + str(_ROOT / "scripts/produce_managed_resources.py"), + "--inputs", + str(prepared_inputs), + "--inputs-sha256", + expected, + "--server-url", + "https://candidate.invalid/server.tar.gz", + "--output", + str(output), + ], + cwd=_ROOT, + check=True, + timeout=600, + ) + return outputs[0], outputs[1], inputs, expected + + +def test_native_preparation_uses_canonical_lock_bytes() -> None: + selected = os.environ.get("POWERCONTEXT_RESOURCE_TEST_INPUTS") + if not selected: + pytest.skip("requires explicitly prepared native pinned inputs") + inputs = Path(selected).resolve() + metadata = json.loads((inputs / "inputs.json").read_bytes()) + content = (inputs / "requirements.lock").read_bytes() + # The consumer compares this hash-bound wire format byte for byte. Windows + # text-mode CRLF translation must not change otherwise identical inputs. + assert content and content.endswith(b"\n") and b"\r" not in content + assert hashlib.sha256(content).hexdigest() == metadata["requirements_sha256"] + + +def test_native_producer_is_deterministic_and_preserves_immutable_output(native_candidates) -> None: + first, second, inputs, expected = native_candidates + for path in ( + "server/server.tar.gz", + "server/catalog.json", + "ops.tar.gz", + "ops/bundle.json", + "ops-provision.json", + "candidate.json", + ): + assert sha256(first / path) == sha256(second / path) + before = sha256(first / "candidate.json") + result = subprocess.run( + [ + sys.executable, + str(_ROOT / "scripts/produce_managed_resources.py"), + "--inputs", + str(inputs), + "--inputs-sha256", + expected, + "--server-url", + "https://candidate.invalid/server.tar.gz", + "--output", + str(first), + ], + cwd=_ROOT, + capture_output=True, + check=False, + ) + assert result.returncode != 0 + assert b"never overwrites" in result.stderr + assert sha256(first / "candidate.json") == before + candidate = json.loads((first / "candidate.json").read_bytes()) + assert candidate["status"] == "unqualified_candidate" + assert candidate["qualification_ref"].startswith("unqualified-candidate:sha256:") + + +def test_native_candidate_is_consumed_and_runs_independent_ops(native_candidates) -> None: + first, _second, _inputs, _expected = native_candidates + subprocess.run( + [ + sys.executable, + str(_ROOT / "scripts/ci_managed_resource_smoke.py"), + "--candidate", + str(first), + "--serve", + "--evidence-dir", + os.environ.get("POWERCONTEXT_RESOURCE_TEST_EVIDENCE", str(first.with_name(first.name + "-evidence"))), + ], + cwd=_ROOT, + check=True, + timeout=300, + ) + + +def test_native_version_mismatch_is_rejected_before_output(native_candidates, tmp_path: Path) -> None: + _first, _second, inputs, _expected = native_candidates + copied = tmp_path / "inputs" + shutil.copytree(inputs, copied, copy_function=shutil.copyfile) + metadata = json.loads((copied / "inputs.json").read_bytes()) + metadata["server_version"] = "999.0.0" + (copied / "inputs.json").write_bytes(json_bytes(metadata)) + output = tmp_path / "rejected" + result = subprocess.run( + [ + sys.executable, + str(_ROOT / "scripts/produce_managed_resources.py"), + "--inputs", + str(copied), + "--inputs-sha256", + sha256(copied / "inputs.json"), + "--server-url", + "https://candidate.invalid/server.tar.gz", + "--output", + str(output), + ], + cwd=_ROOT, + capture_output=True, + check=False, + ) + assert result.returncode != 0 + assert b"requested Server version" in result.stderr + assert not output.exists() diff --git a/uv.lock b/uv.lock index afb36a382e..9776baeb5d 100644 --- a/uv.lock +++ b/uv.lock @@ -2905,6 +2905,30 @@ notebooks = [ { name = "powercontext-langgraph" }, { name = "python-dotenv" }, ] +resource-build = [ + { name = "aiosqlite" }, + { name = "alembic" }, + { name = "apscheduler" }, + { name = "click" }, + { name = "filelock" }, + { name = "httpx" }, + { name = "jsonschema" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "pydantic-settings" }, + { name = "pyobvector" }, + { name = "pytest" }, + { name = "pyyaml" }, + { name = "rich" }, + { name = "ruff" }, + { name = "sqlalchemy", extra = ["asyncio"] }, + { name = "sqlite-vec" }, + { name = "ty" }, + { name = "tzdata" }, + { name = "zstandard" }, +] [package.metadata] requires-dist = [ @@ -3072,6 +3096,30 @@ notebooks = [ { name = "powercontext-langgraph", editable = "integrations/langgraph" }, { name = "python-dotenv", specifier = ">=1,<2" }, ] +resource-build = [ + { name = "aiosqlite", specifier = ">=0.22,<1" }, + { name = "alembic", specifier = ">=1.16,<2" }, + { name = "apscheduler", specifier = ">=3.11,<4" }, + { name = "click", specifier = ">=8.1,<9" }, + { name = "filelock", specifier = ">=3.16,<4" }, + { name = "httpx", specifier = ">=0.28,<1" }, + { name = "jsonschema", specifier = ">=4.23,<5" }, + { name = "opentelemetry-api", specifier = ">=1.30,<2" }, + { name = "opentelemetry-sdk", specifier = ">=1.30,<2" }, + { name = "packaging", specifier = ">=24,<27" }, + { name = "platformdirs", specifier = ">=4,<5" }, + { name = "pydantic-settings", specifier = ">=2.7,<3" }, + { name = "pyobvector", specifier = ">=0.2.28,<0.3" }, + { name = "pytest", specifier = ">=9.0.2" }, + { name = "pyyaml", specifier = ">=6,<7" }, + { name = "rich", specifier = ">=13,<16" }, + { name = "ruff", specifier = ">=0.15.7" }, + { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2,<3" }, + { name = "sqlite-vec", specifier = ">=0.1.9,<0.2" }, + { name = "ty", specifier = ">=0.0.24" }, + { name = "tzdata", specifier = ">=2026.3" }, + { name = "zstandard", specifier = ">=0.23,<1" }, +] [[package]] name = "powercontext-connector-opendal"